首先配置hosts文件关联主机名和ip地址
host1=
host2=
host3=


=== security shell
rm -rf ~/.ssh/*
ssh-keygen  -t rsa -f ~/.ssh/id_rsa  -N ''
ssh-copy-id -o StrictHostKeyChecking=no $remothostname
ssh $remothostname hostname




######################## Hadoop cluster deploy
1. tar -xzf hadoop-2.7.1.tar.gz
2. add profile
Shell> cat << EOF >/etc/profile.d/hadoop.sh
#!/bin/sh
export JAVA_HOME=/root/BIGDATA/jdk1.8.0_65
export HADOOP_PREFIX=/root/BIGDATA/hadoop-2.7.1


export HADOOP_HOME=\$HADOOP_PREFIX
export HADOOP_CONF_DIR=\$HADOOP_HOME/etc/hadoop
export YARN_CONF_DIR=\$HADOOP_HOME/etc/hadoop


export JAVA_LIBRARY_PATH=\$HADOOP_HOME/lib/native:\$JAVA_LIBRARY_PATH
export LD_LIBRARY_PATH=\$HADOOP_HOME/lib/native:\$LD_LIBRARY_PATH
export CLASSPATH=.:\$JAVA_HOME/lib/dt.jar:\$JAVA_HOME/lib/tools.jar:
export PATH=\$JAVA_HOME/bin:\$HADOOP_HOME/bin:\$HADOOP_HOME/sbin:\${PATH}
EOF


Shell> source /etc/profile
 
3. create hdfs dirs on all hosts


HADOOP_LOCAL_BASE_DIR=/opt/local/hdfs

mkdir -p ${HADOOP_LOCAL_BASE_DIR}
mkdir -p ${HADOOP_LOCAL_BASE_DIR}/dfs/data
mkdir -p ${HADOOP_LOCAL_BASE_DIR}/dfs/name
mkdir -p ${HADOOP_LOCAL_BASE_DIR}/dfs/snn
mkdir -p ${HADOOP_LOCAL_BASE_DIR}/tmp
mkdir -p ${HADOOP_LOCAL_BASE_DIR}/yarn/logs


4. config etc/hadoop/
1. add all slaves to slaves
bigdata1
bigdata3


2.
HADOOP_DFS_MASTER=masternode
HADOOP_DFS_SECONDARY_NAMENODE=masternode
YARN_RESOURCE_MANAGER=masternode
JOBHISTORY_SERVER=masternode
JOBTRACKRT_HOST=masternode
HADOOP_TOOL_INSTALL_DIR=/root/BIGDATA/DOCS/hadoop_doc/hadoop_demo
#core-site.xml
conf_file=core-site.xml
cp -raf ${HADOOP_TOOL_INSTALL_DIR}/${conf_file}  ${HADOOP_PREFIX}/etc/hadoop/
sed -i "s^\${HADOOP_LOCAL_BASE_DIR}^${HADOOP_LOCAL_BASE_DIR}^g" "${HADOOP_PREFIX}/etc/hadoop/${conf_file}"
sed -i "s^\${HADOOP_DFS_MASTER}^${HADOOP_DFS_MASTER}^g" "${HADOOP_PREFIX}/etc/hadoop/${conf_file}"
#hdfs-site.xml
conf_file=hdfs-site.xml
cp -raf ${HADOOP_TOOL_INSTALL_DIR}/${conf_file}  ${HADOOP_PREFIX}/etc/hadoop/
sed -i "s^\${HADOOP_LOCAL_BASE_DIR}^${HADOOP_LOCAL_BASE_DIR}^g" "${HADOOP_PREFIX}/etc/hadoop/${conf_file}"
sed -i "s^\${HADOOP_DFS_SECONDARY_NAMENODE}^${HADOOP_DFS_SECONDARY_NAMENODE}^g" "${HADOOP_PREFIX}/etc/hadoop/${conf_file}"
sed -i "s^\${HADOOP_DFS_MASTER}^${HADOOP_DFS_MASTER}^g" "${HADOOP_PREFIX}/etc/hadoop/${conf_file}"
#mapreducesite.xml
conf_file=mapred-site.xml
cp -raf ${HADOOP_TOOL_INSTALL_DIR}/${conf_file}  ${HADOOP_PREFIX}/etc/hadoop/
sed -i "s^\${JOBTRACKRT_HOST}^${JOBTRACKRT_HOST}^g" "${HADOOP_PREFIX}/etc/hadoop/${conf_file}"
sed -i "s^\${JOBHISTORY_SERVER}^${JOBHISTORY_SERVER}^g" "${HADOOP_PREFIX}/etc/hadoop/${conf_file}"
#yarn-site.xml
conf_file=yarn-site.xml
cp -raf ${HADOOP_TOOL_INSTALL_DIR}/${conf_file}  ${HADOOP_PREFIX}/etc/hadoop/
sed -i "s^\${YARN_RESOURCE_MANAGER}^${YARN_RESOURCE_MANAGER}^g" "${HADOOP_PREFIX}/etc/hadoop/${conf_file}"
      sed -i "s^\${HADOOP_LOCAL_BASE_DIR}^${HADOOP_LOCAL_BASE_DIR}^g" "${HADOOP_PREFIX}/etc/hadoop/${conf_file}"






5. init namenode
Shell>hdfs namenode -format cluster1
6. start all
Shell>$HADOOP_HOME/sbin/start-all.sh
Shell> $HADOOP_HOME/sbin/mr-jobhistory-daemon.sh  start historyserver



===Hadoop check
1. After deploy hadoop.
   Shell>hadoop checknative -a 
   Shell>hadoop jar ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar pi 4 100
   
   Shell> cat <<EOF >/tmp/file1
Hello World Bye World
EOF
   Shell> cat <<EOF >/tmp/file2
Hello Hadoop Goodbye Hadoop
EOF

   Shell> hadoop fs -mkdir /tmp 
   Shell> hadoop fs -copyFromLocal -f /tmp/file1  /tmp/file2  /tmp
   Shell> hadoop jar ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar wordcount  /tmp/file1  /tmp/file2  /tmp/wordcount
   Shell> hadoop fs -cat /tmp/wordcount/part-r-00000




===hadoop Daemon Web Interface
NameNode http://nn_host:port/ Default HTTP port is 50070.
ResourceManager http://rm_host:port/ Default HTTP port is 8088.
#MapReduce JobHistory Server http://jhs_host:port/ Default HTTP port is 19888.




######################## Spark cluster deploy
1. tar -xzf spark-1.6.1-bin-hadoop2.6.tgz
2. add profile
cat << EOF >>/etc/profile.d/hadoop.sh
export SPARK_HOME=/root/BIGDATA/spark-1.6.1-bin-hadoop2.6
export PATH=\${SPARK_HOME}/sbin:\${PATH}:\${SPARK_HOME}/bin:


EOF


Shell>source /etc/profile

3. create local dir
SPARK_LOCAL_BASE_DIR=/opt/local/spark

Shell>mkdir -p ${SPARK_LOCAL_BASE_DIR}/tmp

Shell>hadoop fs -mkdir /sparkHistoryLogs /sparkEventLogs
4. config
1. add all slaves to slaves
       Shell>mv slaves.template slaves
bigdata1
bigdata3

2.
SPARK_MASTER=masternode
HADOOP_DFS_MASTER=masternode

Shell> cat << EOF > ${SPARK_HOME}/conf/spark-defaults.conf
spark.master   spark://${SPARK_MASTER}:7077
spark.local.dir   ${SPARK_LOCAL_BASE_DIR}/tmp
spark.master.rest.port   7177
#Spark UI
spark.eventLog.enabled   true
spark.eventLog.dir   hdfs://${HADOOP_DFS_MASTER}:9000/sparkEventLogs
spark.ui.killEnabled   true
spark.ui.port   4040
spark.history.ui.port   18080
spark.history.fs.logDirectory   hdfs://${HADOOP_DFS_MASTER}:9000/sparkHistoryLogs


#
spark.shuffle.service.enabled   false


#
spark.yarn.am.extraJavaOptions   -Xmx3g
spark.executor.extrajavaoptions   -Xmx3g


#Amount of memory to use for the YARN Application Master in client mode
spark.yarn.am.memory   2048m
#The amount of off-heap memory (in megabytes) to be allocated per executor. 
spark.yarn.driver.memoryOverhead   512
#The amount of off-heap memory (in megabytes) to be allocated per driver in cluster mode
spark.yarn.executor.memoryOverhead   512
#Same as spark.yarn.driver.memoryOverhead, but for the YARN Application Master in client mode, fix yarn-client OOM, "ERROR yarn.ApplicationMaster: RECEIVED SIGNAL 15: SIGTERM"
spark.yarn.am.memoryOverhead   1024  
  
EOF


Shell> cat << EOF > ${SPARK_HOME}/conf/spark-env.sh
SPARK_WORKER_WEBUI_PORT=8081
SPARK_WORKER_DIR=\${SPARK_HOME}/work
#SPARK_LOCAL_DIRS=\${SPARK_WORKER_DIR}
EOF


5. start all
Shell> ${SPARK_HOME}/sbin/start-all.sh
check cluster status
http://${SPARK_MASTER}:8080


===Spark Daemon Web Interface
spark.history.ui.port 18080
spark master 8080


http://${SPARK_MASTER}:port/




===Spark check


1. Spark Standalone (client, cluster(spark.master.rest.port))
  # Run application locally on 1 cores
  Shell>  ${SPARK_HOME}/bin/spark-submit \
  --class org.apache.spark.examples.SparkPi \
  --master spark://masternode:7077 \
  --deploy-mode  client \
   ${SPARK_HOME}/lib/spark-examples*.jar \
  10


  # Run on a Spark standalone cluster
  Shell>  ${SPARK_HOME}/bin/spark-submit \
  --class org.apache.spark.examples.SparkPi \
  --master spark://$SPARK_MASTER:7177 \
  --deploy-mode  cluster \
  --executor-memory 1G \
  --total-executor-cores 1 \
   ${SPARK_HOME}/lib/spark-examples*.jar \
  10
  
   #spark shell
   Shell> ${SPARK_HOME}/bin/spark-shell --master spark://$SPARK_MASTER:7077
   


2. Spark on Yarn (It needn't start spark cluster, only need start hadoop)
#run yarn-client
Shell> ${SPARK_HOME}/bin/spark-submit --class org.apache.spark.examples.SparkPi \
--master yarn-client \
    --driver-java-options '-Xmx3096m'  \
    --conf spark.executor.extrajavaoptions=-Xmx3096m  \
    --executor-memory 3096m  \
    --num-executors  1  \
    --conf spark.yarn.am.memoryOverhead=1024  \
    ${SPARK_HOME}/lib/spark-examples*.jar \
    10
    

#run yarn-cluster
Shell> ${SPARK_HOME}/bin/spark-submit --class org.apache.spark.examples.SparkPi \
--master yarn \
--deploy-mode  cluster \
    --driver-memory 2g \
    --executor-memory 2g \
    ${SPARK_HOME}/lib/spark-examples*.jar \
    10




######################## Hbase cluster deploy
1. Shell> tar -xzf hbase-1.1.4-bin.tar.gz
2. add profile
cat << EOF >>/etc/profile.d/hadoop.sh
export HBASE_HOME=/root/BIGDATA/hbase-1.1.4
export PATH=\${PATH}:\${HBASE_HOME}/bin:


EOF


Shell>source /etc/profile

3. create local dir
HBASE_ROOTDIR=/hbase
HBASE_TMP_DIR=/opt/local/hbase

Shell> hadoop fs -mkdir ${HBASE_ROOTDIR}
Shell> mkdir -p ${HBASE_TMP_DIR}
4. config
1. add all hosts to regionservers
bigdata1
bigdata2

2. modify hbase-site.xml
cat <<EOF >${HBASE_HOME}/conf/hbase-site.xml
<configuration>
  <property>
    <name>hbase.rootdir</name>
    <value>hdfs://masternode:9000/hbase </value>
    <description>The directory shared by RegionServers.
    Default: \${hbase.tmp.dir}/hbase
    </description>
  </property>
  <property>
    <name>hbase.zookeeper.quorum</name>
    <value>masternode,slavesnode</value>
    <description>The directory shared by RegionServers.
    </description>
  </property>
  <property>
    <name>hbase.tmp.dir</name>
    <value>/opt/local/hbase</value>
    <description>Temporary directory on the local filesystem
    Default: \${java.io.tmpdir}/hbase-${user.name}.
    </description>
  </property>
  <property>
    <name>hbase.cluster.distributed</name>
    <value>true</value>
    <description>The mode the cluster will be in. Possible values are
      false: standalone and pseudo-distributed setups with managed Zookeeper
      true: fully-distributed with unmanaged Zookeeper Quorum (see hbase-env.sh)
    </description>
  </property>
  <!--
  <property>
    <name>hbase.fs.tmp.dir</name>
    <value></value>
    <description>A staging directory in default file system (HDFS) for keeping temporary data
    Default: /user/\${user.name}/hbase-staging
    </description>
  </property>
  <property>
    <name>hbase.local.dir</name>
    <value></value>
    <description>Directory on the local filesystem to be used as a local storage.
    Default: ${hbase.tmp.dir}/local/
    </description>
  </property>
  <property>
    <name>hbase.master.port</name>
    <value>16000</value>
    <description>The port the HBase Master should bind to.
    Default: 16000
    </description>
  </property>
  <property>
    <name>hbase.master.info.port</name>
    <value>16010</value>
    <description>The port for the HBase Master web UI. Set to -1 if you do not want a UI instance run.
    Default: 16010
    </description>
  </property>
  <property>
    <name>hbase.regionserver.port</name>
    <value>16020</value>
    <description>The port the HBase RegionServer binds to.
    Default: 16020
    </description>
  </property>
  <property>
    <name>hbase.regionserver.info.port</name>
    <value>16030</value>
    <description>The port for the HBase RegionServer web UI Set to -1 if you do not want the RegionServer UI to run.
    Default: 16030
    </description>
  </property>
  <property>
    <name>hbase.zookeeper.peerport</name>
    <value>2888</value>
    <description>Port used by ZooKeeper peers to talk to each other
    Default: 2888
    </description>
  </property>
  <property>
    <name>hbase.zookeeper.leaderport</name>
    <value>3888</value>
    <description>Port used by ZooKeeper for leader election
    Default: 3888
    </description>
  </property>
  <property>
    <name>hbase.zookeeper.property.dataDir</name>
    <value></value>
    <description>Property from ZooKeeper’s config zoo.cfg. The directory where the snapshot is stored.
    Default: ${hbase.tmp.dir}/zookeeper
    </description>
  </property>
  <property>
    <name>hbase.zookeeper.property.clientPort</name>
    <value>2181</value>
    <description>Property from ZooKeeper’s config zoo.cfg. The port at which the clients will connect.
    Default: 2181
    </description>
  </property>
  -->
  
</configuration>
EOF
3. ln -s $HADOOP_HOME/etc/hadoop/hdfs-site.xml  ${HBASE_HOME}/conf/hdfs-site.xml 
4. ulimit 咿nproc
cat <<EOF > /etc/security/limits.conf
 root -       nofile  32768
 root soft/hard nproc 32000
EOF

5. start all
Shell> ${HBASE_HOME}/bin/start-hbase.sh


===Hbase Daemon Web Interface
hbase.master.info.port  16010
hbase.regionserver.info.port  16030


http://${HBASE_MASTER}:port/


===Hbase check


1. run hbase shell
Shell> ${HBASE_HOME}/bin/hbase shell


hbase(main):003:0> create 'test', 'cf'
0 row(s) in 1.2200 seconds
hbase(main):003:0> list 'table'
test
1 row(s) in 0.0550 seconds
hbase(main):004:0> put 'test', 'row1', 'cf:a', 'value1'
0 row(s) in 0.0560 seconds
hbase(main):005:0> put 'test', 'row2', 'cf:b', 'value2'
0 row(s) in 0.0370 seconds
hbase(main):006:0> put 'test', 'row3', 'cf:c', 'value3'
0 row(s) in 0.0450 seconds


hbase(main):007:0> scan 'test'
ROW        COLUMN+CELL
row1       column=cf:a, timestamp=1288380727188, value=value1
row2       column=cf:b, timestamp=1288380738440, value=value2
row3       column=cf:c, timestamp=1288380747365, value=value3
3 row(s) in 0.0590 seconds


hbase(main):008:0> get 'test', 'row1'
COLUMN      CELL
cf:a        timestamp=1288380727188, value=value1
1 row(s) in 0.0400 seconds


hbase(main):012:0> disable 'test'
0 row(s) in 1.0930 seconds
hbase(main):013:0> drop 'test'
0 row(s) in 0.0770 seconds 


hbase(main):014:0> exit




######################## Hive cluster deploy
1. tar -xzf apache-hive-2.0.0-bin.tar.gz
2. add profile
cat << EOF >>/etc/profile.d/hadoop.sh
export HIVE_HOME=/root/BIGDATA/apache-hive-2.0.0-bin
export HIVE_CONF_DIR=\${HIVE_HOME}/conf
export PATH=\${HIVE_HOME}/bin:\${PATH}


EOF


Shell>source /etc/profile

3. create local dir
   $HADOOP_HOME/bin/hadoop fs -mkdir /tmp
   $HADOOP_HOME/bin/hadoop fs -mkdir -p /user/hive/warehouse
   $HADOOP_HOME/bin/hadoop fs -chmod g+w  /tmp
   $HADOOP_HOME/bin/hadoop fs -chmod g+w  /user/hive/warehouse

   Shell> mkdir -p  ${HBASE_TMP_DIR}

4. config 
=M1. [ Local Embedded Derby ]
HIVE_LOCAL_WAREHOUSE=/opt/hive/warehouse
Shell> mkdir -p  ${HIVE_LOCAL_WAREHOUSE}

Shell>cat <<EOF > ${HIVE_CONF_DIR}/hive-site.xml
<configuration>
<property>
  <name>javax.jdo.option.ConnectionURL</name>
  <value>jdbc:derby:;databaseName=metastore_db;create=true</value>
  <description>JDBC connect string for a JDBC metastore</description>
</property>


<property>
  <name>javax.jdo.option.ConnectionDriverName</name>
  <value>org.apache.derby.jdbc.EmbeddedDriver</value>
  <description>Driver class name for a JDBC metastore</description>
</property>


<property>
  <name>javax.jdo.option.ConnectionUserName</name>
  <value>APP</value>
  <description>username to use against metastore database</description>
</property>


<property>
  <name>javax.jdo.option.ConnectionPassword</name>
  <value>mine</value>
  <description>password to use against metastore database</description>
</property>


<property>
  <name>hive.metastore.warehouse.dir</name>
  <value>${HIVE_LOCAL_WAREHOUSE}</value>
  <description>unit test data goes in here on your local filesystem</description>
</property>


</configuration>
EOF


Shell> $HIVE_HOME/bin/schematool -initSchema -dbType derby
Shell> $HIVE_HOME/bin/schematool -dbType derby-info
Shell> $HIVE_HOME/bin/hive





=M2. [Remote Metastore Server Derby]
Shell> tar -xzf db-derby-10.12.1.1-bin.tar.gz
Shell> cd db-derby-10.12.1.1-bin
Shell> mkdir data
Shell> cd data
Shell> ../bin/startNetworkServer  -h 172.31.200.110 -p 1527  &
Shell> cp -raf  ../lib/derbyclient.jar   ../lib/derbytools.jar  $HIVE_HOME/lib/

DERBY_SERVER_HOST=masternode


Shell>cat <<EOF > ${HIVE_CONF_DIR}/hive-site.xml
<configuration>
<property>
  <name>javax.jdo.option.ConnectionURL</name>
  <value>jdbc:derby://${DERBY_SERVER_HOST}:1527/hive_meta;create=true</value>
  <description>JDBC connect string for a JDBC metastore</description>
</property>


<property>
  <name>javax.jdo.option.ConnectionDriverName</name>
  <value>org.apache.derby.jdbc.ClientDriver</value>
  <description>Driver class name for a JDBC metastore</description>
</property>


<property>
    <name>datanucleus.schema.autoCreateAll</name>
    <value>true</value>
    <description>creates necessary schema on a startup if one doesn't exist. set this to false, after creating it once</description>
</property>


<property>
  <name>javax.jdo.option.ConnectionUserName</name>
  <value>app</value>
  <description>username to use against metastore database</description>
</property>


<property>
  <name>javax.jdo.option.ConnectionPassword</name>
  <value>app</value>
  <description>password to use against metastore database</description>
</property>


<property>
  <name>hive.metastore.warehouse.dir</name>
  <!-- base hdfs path -->
  <value>/user/hive/warehouse</value>
  <description>base hdfs path :location of default database for the warehouse</description>
</property>
  
<!-- hive client -->
 <!-- thrift://<host_name>:<port> -->
 <property>
      <name>hive.metastore.uris</name>
      <value>thrift://masternode:9083</value>
 </property>
</configuration>
EOF


#start metastore service
$HIVE_HOME/bin/hive --service metastore &


#star thiveserver service
$HIVE_HOME/bin/hiveserver2 &


5. start
$HIVE_HOME/bin/hive
hive> CREATE TABLE pokes (foo INT, bar STRING);
hive> CREATE TABLE invites (foo INT, bar STRING) PARTITIONED BY (ds STRING);
hive> SHOW TABLES;
hive> SHOW TABLES '.*s';
hive> DESCRIBE invites;

hive> LOAD DATA LOCAL INPATH '/root/BIGDATA/apache-hive-2.0.0-bin/examples/files/kv1.txt' OVERWRITE INTO TABLE pokes;

======#
#Remote Metastore Server
   $HIVE_HOME/bin/hive --service metastore -p 9083
#Running HiveServer2 and Beeline
   $HIVE_HOME/bin/hiveserver2
   $HIVE_HOME/bin/beeline -u jdbc:hive2://localhost:10000
#Running HCatalog
   $HIVE_HOME/hcatalog/sbin/hcat_server.sh
   $HIVE_HOME/hcatalog/bin/hcat
#Running WebHCat
   $HIVE_HOME/hcatalog/sbin/webhcat_server.sh






####### pig
2. add profile
cat << EOF >>/etc/profile.d/hadoop.sh
export PIG_HOME=/BIGDATA/pig-0.15.0
export PATH=\${PIG_HOME}/bin:\${PATH}


EOF


Shell>source /etc/profile