-
Notifications
You must be signed in to change notification settings - Fork 4
/
Copy pathtemp
51 lines (36 loc) · 4.96 KB
/
temp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
mysql -u root -phadoop
create database employees;
grant all privileges on employees.* to 'sqoop'@'localhost' identified by 'hadoop';
flush privileges;
mysql -u sqoop -phadoop employees < employees.sql
mysql -u sqoop -phadoop employees
show tables;
sqoop import --connect jdbc:mysql://localhost:3306/employees --driver com.mysql.jdbc.Driver --username sqoop --password hadoop --table employees --target-dir '/stage-data/employees' --split-by 1
sqoop import --connect jdbc:mysql://localhost:3306/employees --driver com.mysql.jdbc.Driver --username sqoop --password hadoop --table current_dept_emp --target-dir '/stage-data/employees/current_dept_emp' --split-by 1
sqoop import --connect jdbc:mysql://localhost:3306/employees --driver com.mysql.jdbc.Driver --username sqoop --password hadoop --table departments --target-dir '/stage-data/employees/departments' --split-by 1
sqoop import --connect jdbc:mysql://localhost:3306/employees --driver com.mysql.jdbc.Driver --username sqoop --password hadoop --tabldept_emp --target-dir '/stage-data/employees/dept_emp' --split-by 1
sqoop import --connect jdbc:mysql://localhost:3306/employees --driver com.mysql.jdbc.Driver --username sqoop --password hadoop --tabldept_emp_latest_date --target-dir '/stage-data/employees/dept_emp_latest_date' --split-by 1
sqoop import --connect jdbc:mysql://localhost:3306/employees --driver com.mysql.jdbc.Driver --username sqoop --password hadoop --tabldept_manager --target-dir '/stage-data/employees/dept_manager' --split-by 1
sqoop import --connect jdbc:mysql://localhost:3306/employees --driver com.mysql.jdbc.Driver --username sqoop --password hadoop --tablsalaries --target-dir '/stage-data/employees/salaries' --split-by 1
CREATE EXTERNAL TABLE `access`(
`host` string,
`identity` string,
`userid` string,
`accesstime` string,
`request` string,
`status` string,
`size` string, `referer` string, `agent` string )
PARTITIONED BY ( etl_ymd string )
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ' '
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION
'hdfs://sandbox-hdp.hortonworks.com:8020/stage-data/weblog/access/';
# python pip install
hadoop fs -touchz /stage-data/weblog/access/20180401/_SUCCESS /stage-data/weblog/access/20180402/_SUCCESS /stage-data/weblog/access/20180403/_SUCCESS /stage-data/weblog/access/20180404/_SUCCESS /stage-data/weblog/access/20180405/_SUCCESS /stage-data/weblog/access/20180406/_SUCCESS /stage-data/weblog/access/20180407/_SUCCESS /stage-data/weblog/access/20180408/_SUCCESS /stage-data/weblog/access/20180409/_SUCCESS /stage-data/weblog/access/20180410/_SUCCESS
hadoop fs -touchz /stage-data/weblog/access/20180411/_SUCCESS /stage-data/weblog/access/20180412/_SUCCESS /stage-data/weblog/access/20180413/_SUCCESS /stage-data/weblog/access/20180414/_SUCCESS /stage-data/weblog/access/20180415/_SUCCESS /stage-data/weblog/access/20180416/_SUCCESS /stage-data/weblog/access/20180417/_SUCCESS /stage-data/weblog/access/20180418/_SUCCESS /stage-data/weblog/access/20180419/_SUCCESS /stage-data/weblog/access/20180420/_SUCCESS
hadoop fs -touchz /stage-data/weblog/access/20180421/_SUCCESS /stage-data/weblog/access/20180422/_SUCCESS /stage-data/weblog/access/20180423/_SUCCESS /stage-data/weblog/access/20180424/_SUCCESS /stage-data/weblog/access/20180425/_SUCCESS /stage-data/weblog/access/20180426/_SUCCESS /stage-data/weblog/access/20180427/_SUCCESS /stage-data/weblog/access/20180428/_SUCCESS /stage-data/weblog/access/20180429/_SUCCESS /stage-data/weblog/access/20180430/_SUCCESS
hadoop fs -touchz /stage-data/weblog/access/20180501/_SUCCESS /stage-data/weblog/access/20180502/_SUCCESS /stage-data/weblog/access/20180503/_SUCCESS /stage-data/weblog/access/20180504/_SUCCESS /stage-data/weblog/access/20180505/_SUCCESS /stage-data/weblog/access/20180506/_SUCCESS /stage-data/weblog/access/20180507/_SUCCESS /stage-data/weblog/access/20180508/_SUCCESS /stage-data/weblog/access/20180509/_SUCCESS