ECS集群部署大数据环境

bind() {
    echo $1 > /sys/bus/pci/drivers/iohub_sriov/unbind
    echo $1 > /sys/bus/pci/drivers/virtio-pci/bind
}
ubind() {
    echo $1 > /sys/bus/pci/drivers/virtio-pci/unbind
    echo $1 > /sys/bus/pci/drivers/iohub_sriov/bind
}
main() {
    local op=$1
    local bdf=$2
    if [ $bdf == "all" ]; then
    for i in `xdragon-bdf net all`
    do
        eval $op $i
    done
    else
        eval $op $bdf
    fi
}
main $@

bind() {
    echo $1 > /sys/bus/pci/drivers/iohub_sriov/unbind
    echo $1 > /sys/bus/pci/drivers/virtio-pci/bind
}
ubind() {
    echo $1 > /sys/bus/pci/drivers/virtio-pci/unbind
    echo $1 > /sys/bus/pci/drivers/iohub_sriov/bind
}
main() {
    local op=$1
    local bdf=$2
    if [ $bdf == "all" ]; then
    for i in `xdragon-bdf blk  all`
    do
        eval $op $i
    done
    else
        eval $op $bdf
    fi
}
main $@

mkfs -t ext4 /dev/vdb
mkfs.ext4 /dev/vdc
mkfs.ext4 /dev/vdd
mkfs.ext4 /dev/vde
mkfs.ext4 /dev/vdf
mkfs.ext4 /dev/vdg
mkfs.ext4 /dev/vdh
mkfs.ext4 /dev/vdi
mkfs.ext4 /dev/vdj
mkfs.ext4 /dev/vdk

mount /dev/vdb /mnt/disk1
mount /dev/vdc /mnt/disk2
mount /dev/vdd /mnt/disk3
mount /dev/vde /mnt/disk4
mount /dev/vdf /mnt/disk5
mount /dev/vdg /mnt/disk6
mount /dev/vdh /mnt/disk7
mount /dev/vdi /mnt/disk8
mount /dev/vdj /mnt/disk9
mount /dev/vdj /mnt/disk10

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Default system properties included when running spark-submit.
# This is useful for setting default environmental settings.
# Example:
# spark.master                     spark://master1:7077
# spark.eventLog.enabled           true
# spark.eventLog.dir               hdfs://namenode:8021/directory
# spark.serializer                 org.apache.spark.serializer.KryoSerializer
# spark.driver.memory              5g
# spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"
spark.master yarn
spark.deploy-mode client
#driver
spark.driver.cores 4
spark.driver.memory 10g
spark.driver.maxResultSize 10g
##executor
spark.executor.instances 45
spark.executor.memory 13g
spark.executor.cores 4
#shuffle
spark.task.maxFailures 4
spark.default.parallelism  180
spark.sql.shuffle.partitions 180
spark.shuffle.compress true
spark.shuffle.spill.compress true
#other
spark.task.maxFailures 4
spark.kryoserializer.buffer 640k
spark.memory.storageFraction 0.5
spark.shuffle.file.buffer 32k
spark.kryoserializer.buffer.max 2000m
spark.serializer org.apache.spark.serializer.KryoSerializer
spark.memory.fraction 0.6
spark.network.timeout 3600
spark.sql.broadcastTimeout 3600
spark.locality.wait=0s
#speculation
#spark.speculation=true
#spark.speculation.interval=300s
#spark.speculation.quantile=0.9
#spark.speculation.multiplier=1.5
#aqe
spark.sql.adaptive.enabled true
spark.sql.autoBroadcastJoinThreshold 128m
spark.sql.adaptive.advisoryPartitionSizeInBytes 128MB
spark.sql.adaptive.coalescePartitions.minPartitionNum 1
spark.sql.adaptive.coalescePartitions.initialPartitionNum 180
spark.sql.adaptive.forceApply true
spark.sql.adaptive.coalescePartitions.enabled   true
spark.sql.adaptive.localShuffleReader.enabled   true
spark.sql.adaptive.skewJoin.enabled     true
spark.sql.adaptive.skewJoin.skewedPartitionFactor 5
spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes 256m
#DF
#spark.sql.optimizer.dynamicPartitionPruning.enabled  false
#spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcastOnly true
spark.sql.optimizer.dynamicDataPruning.pruningSideThreshold 10GB
#spark.sql.optimizer.dynamicDataPruning.enabled false
#cbo
#spark.sql.statistics.histogram.enabled true
#spark.sql.statistics.histogram.numBins 32
spark.sql.cbo.enabled true
spark.sql.cbo.joinReorder.enabled true
spark.sql.cbo.planStats.enabled  true
spark.sql.cbo.starSchemaDetection true
spark.sql.cbo.joinReorder.card.weight  0.6
spark.sql.cbo.joinReorder.ga.enabled true
spark.sql.autoBroadcastJoinRowThreshold 500000
#log
spark.eventLog.enabled true
spark.eventLog.dir hdfs://master1:9000/sparklogs
spark.eventLog.compress true
log4j.logger.org.apache.storage.ShuffleBlockFetcherIterator TRACE

cd /usr/local/mysql/support-files/
touch my-default.cnf
chmod 777 my-default.cnf
cp /usr/local/mysql/support-files/my-default.cnf /etc/my.cnf
vi /etc/my.cnf
#my.cnf中添加：
---------------------------------------------------------------------------------------------------------------------------------------------
# Remove leading # and set to the amount of RAM for the most important data
# cache in MySQL. Start at 70% of total RAM for dedicated server, else 10%.
# innodb_buffer_pool_size = 128M
# Remove leading # to turn on a very important data integrity option: logging
# changes to the binary log between backups.
# log_bin
# These are commonly set, remove the # and set as required.
[mysqld]
basedir = /usr/local/mysql
datadir = /usr/local/mysql/data
socket = /tmp/mysql.sock
#socket =/var/lib/mysql/mysql.socket
log-error = /usr/local/mysql/data/error.log
pid-file = /usr/local/mysql/data/mysql.pid
user = root
tmpdir = /tmp
port = 3306
#skip-grant-tables
#lower_case_table_names = 1
# server_id = .....
# socket = .....
#lower_case_table_names = 1
max_allowed_packet=32M
default-authentication-plugin = mysql_native_password
#lower_case_file_system = on
#lower_case_table_names = 1
log_bin_trust_function_creators = ON
# Remove leading # to set options mainly useful for reporting servers.
# The server defaults are faster for transactions and fast SELECTs.
# Adjust sizes as needed, experiment to find the optimal values.
# join_buffer_size = 128M
# sort_buffer_size = 2M
# read_rnd_buffer_size = 2M
sql_mode=NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES

###UBUNTU
echo 'deb http://archive.ubuntu.com/ubuntu/ trusty main universe restricted multiverse' >>/etc/apt/sources.list
sudo apt-get update
sudo apt-get install sysv-rc-conf
sysv-rc-conf --list

mysql> alter user user() identified by "123456";
Query OK, 0 rows affected (0.01 sec)
mysql> ALTER user 'root'@'localhost' IDENTIFIED BY '123456';
Query OK, 0 rows affected (0.00 sec)
mysql> use mysql;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A
Database changed
mysql> update user set host = '%' where user = 'root';
Query OK, 1 row affected (0.00 sec)
Rows matched: 1  Changed: 1  Warnings: 0
mysql> FLUSH PRIVILEGES;
Query OK, 0 rows affected (0.00 sec)
用户授权
grant all privileges on  *.* to 'hive'@'%';

<configuration>
    <property>
        <name>javax.jdo.option.ConnectionUserName</name>
        <value>root</value>
    </property>
    <property>
        <name>javax.jdo.option.ConnectionPassword</name>
        <value>root</value>
    </property>
   <property>
        <name>javax.jdo.option.ConnectionURL</name>
        <value>jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true</value>
    </property>
  <property>
        <name>javax.jdo.option.ConnectionDriverName</name>
        <value>com.mysql.cj.jdbc.Driver</value>
    </property>
 <property>
    <name>datanucleus.schema.autoCreateAll</name>
    <value>true</value>
  </property>
  <property>
                <name>hive.metastore.schema.verification</name>
                <value>false</value>
        </property>
</configuration>

mysql> create user 'hive'@'localhost' identified by '123456';
Query OK, 0 rows affected (0.00 sec)
mysql> grant all privileges on *.* to 'hive'@'localhost';
Query OK, 0 rows affected (0.00 sec)
mysql> create user 'hive'@'%' identified by '123456';
Query OK, 0 rows affected (0.00 sec)
mysql> grant all privileges on *.* to 'hive'@'%';
Query OK, 0 rows affected (0.00 sec)
mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)