Sqoop用法

// 最常用的sqooop命令: 可以根據(jù)help查看各種參數(shù)
sqoop help 

sqoop help list-databases

// 導(dǎo)入數(shù)據(jù)到hdfs
sqoop import --connect jdbc:mysql://localhost:3306/test_db --username bigdata --password hq.2020.DB --table tb_user

// hdfs查看當(dāng)前用戶下的文件
hdfs dfs -ls

``


# Sqoop的安裝和部署

安裝sqoop,并創(chuàng)建軟連接
ln -s sqoop-1.4.6-cdh5.16.2 sqoop-release
// 更新軟連接;
ln -snf sqoop-1.4.6-cdh5.16.3 sqoop-release

下載和hadoop cdh后綴一樣的sqoop;
修改配置文件:
vim conf/sqoop-env.sh

export HADOOP_HOME=/home/bigdata/app/hadoop-release
export HADOOP_COMMON_HOME=${HADOOP_HOME}

export HADOOP_MAPRED_HOME=${HADOOP_HOME}
export HBASE_HOME=/home/bigdata/app/hbase-release
export HIVE_HOME=/home/bigdata/app/hive-release
export ZOOCFGDIR=/home/bigdata/app/zookeeper-release

2. 加上 mysql的jdbc
cp mysql-connector-java-5.1.47.jar $SQOOP_HOME/lib





# 報錯分析

### org.json.JSONObject 缺失

20/06/13 16:28:19 INFO mapreduce.ImportJobBase: Beginning import of tb_user
Exception in thread "main" java.lang.NoClassDefFoundError: org/json/JSONObject
at org.apache.sqoop.util.SqoopJsonUtil.getJsonStringforMap(SqoopJsonUtil.java:43)
at org.apache.sqoop.SqoopOptions.writeProperties(SqoopOptions.java:784)
at org.apache.sqoop.mapreduce.JobBase.putSqoopOptionsToConfiguration(JobBase.java:392)
at org.apache.sqoop.mapreduce.JobBase.createJob(JobBase.java:378)
at org.apache.sqoop.mapreduce.ImportJobBase.runImport(ImportJobBase.java:256)
at org.apache.sqoop.manager.SqlManager.importTable(SqlManager.java:691)
at org.apache.sqoop.manager.MySQLManager.importTable(MySQLManager.java:127)
at org.apache.sqoop.tool.ImportTool.importTable(ImportTool.java:513)
at org.apache.sqoop.tool.ImportTool.run(ImportTool.java:621)
at org.apache.sqoop.Sqoop.run(Sqoop.java:147)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)
at org.apache.sqoop.Sqoop.main(Sqoop.java:252)
Caused by: java.lang.ClassNotFoundException: org.json.JSONObject
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 15 more

缺失 json-20160810.jar  這個jar包:

<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20180813</version>
</dependency>

//解決辦法: 從其他組件的依賴中尋找 json-*.jar的包,復(fù)制過去;
find -name "json"
./zeppelin-0.8.2-bin-all/lib/json-smart-2.3.jar
./zeppelin-0.8.2-bin-all/lib/jersey-media-json-jackson-2.27.jar
./zeppelin-0.8.2-bin-all/lib/json-20160810.jar
// 復(fù)制該 json-.jar 到sqoop/lib下;
cp zeppelin-0.8.2-bin-all/lib/json-20160810.jar $SQOOP_HOME/lib