hadoop fs -mkdir mydir
hadoop fs -ls
hadoop fs -cat myfile
hadoop fs -du
hadoop fs -expunge
hadoop fs -chgrp hadoop file1
hadoop fs -chown huser file1
hadoop fs -rm file1
hadoop fs -touchz file2
hadoop fs -stat file1
hadoop fs -test -e file1
hadoop fs -test -z file1
hadoop fs -test -d file1
hadoop fs -copyFromLocal <source> <destination>
hadoop fs -copyFromLocal file1 data
hadoop fs -copyToLocal <source> <destination>
hadoop fs -copyToLocal data/file1 /var/tmp
hadoop fs -put <source> <destination>
hadoop fs -get <source> <destination>
hadoop distcp hdfs://[Link]:8020/input hdfs://[Link]:8020/output
hadoop fs -mv [Link] /user/hduser/data
hadoop fs -setrep -w 4 file1
hadoop fs -getmerge mydir bigfile
Hive Table Commands:
create database dbname;
create [Link];
use dbname;
show tables;
desc tablename;
desc formatted tablename;
show create table tablename;
truncate table tablename;
drop table tablename;
drop database [if exists] databasename;
drop database [if exists] databasename cascade;
show transactions;
show compactions;
show partitions [Link];
alter table tablename rename to newtablename;
alter table tablename replace colname1 colname2;
Internal/Managed table:
CREATE TABLE [if exists] TBLNAME(COL DTYPE,COL DTYPE...)
alter table tablename change [columname oldvalue newvalue];
insert into tablename values(x,y,z);
CREATE TABLE [if exists] TBLNAME(COL DTYPE,COL DTYPE...)
COMMENT 'TESTING'
ROW FORMATTED
FIELDS DELIMITED BY ':'
LINES TERMINATED BY '\N'
STORED AS 'PARQUET';
CREATE EXTERNAL TABLE TBLNAME(COL DTYPE,COL DTYPE...)
COMMENT 'TESTING'
ROW FORMATTED
FIELDS DELIMITED BY ','
LINES TERMINATED BY '\N'
STORED AS 'TEXTFILE'
LOCATION 'HDFS PATH';
LOAD DATA INPATH 'HDFS PATH' INTO TABLE TABLENAME
LOAD DATA LOCAL INPATH 'UNIX PATH' INTO TABLE TABLENAME;
Insert into tablename select * from tablename; (Pre-requisite: Create table
first)
Insert append into tablename select * from tablename;
Create table as select * from tablename; (CTAS)
create external table tablename(colname1 dtype,colname2 dtype)
PARTITIONED BY (colname3 dtype)
CLUSTERED BY (colname2) into X buckets;
create external table tablename(colname1 dtype,colname2 dtype)
CLUSTERED BY (colname2) into 4 buckets;
TBLPROPERTIES('[Link]'='true');
create external table tablename(colname1 dtype,colname2 dtype)
PARTITIONED BY (colname3 dtype)
CLUSTERED BY (colname2) into X buckets
TBLPROPERTIES('transactions'='true');
TBLPROPERTIES('transactional'='true');
create database ${hiveconf:reference}_hive;
hive -f '[Link]'
hive -e 'select * from test.test_internal;' >> '/home/cloudera/test_internal.txt'
!connect jdbc:hive2://localhost:10000/test
Hbase Table Commands:
list
describe
status
help
version
whoami
alter
create
disable
disable_all
is_disabled
drop
drop_all
enable
enable_all
is_enabled
exists
put
scan
truncate
count
delete
deleteall
get
show_filters
alter_status
alter_async
get_counter
incr
assign
balancer
balance_switch
close_region
compact
Flush
major_compact
move
split
unassign
hlog_roll
zk_dump
add_peer
remove_peer
list_peers
enable_peer
disable_peer
start_replication
stop_replication
grant
revoke
user_permission
Pig Commands:
LINES = LOAD '/user/cloudera/xxx/input/Word_Count_input.txt' using
PigStorage(',') AS (line:chararray);
WORDS = FOREACH lines GENERATE FLATTEN(TOKENIZE(line)) as word;
GROUPED = GROUP words BY word;
WORDCOUNT = FOREACH grouped GENERATE group, COUNT(words);
DUMP WORDCOUNT;