Hbase manual compact and split

#! / bin/bashdie () {echo > & 2 "$@" echo "usage:" echo "$0 check | split table_name [split_size]" exit 1} [["$#"-lt 2]] & & die "at least 2 arguments required, $# provided" COMMAND=$1TABLE=$2SIZE= $3mur1073741824} "split () {region_key= `python / home/hduser/hbase/hbase-scan.py-t hbase:meta-f" RowFilter (= 'substring:$1') "`echo" split' $region_key' "| hbase shell} if [" $COMMAND "! =" check "] Then for region in `hadoop fs-ls / hbase/data/default/$TABLE | awk {'print $8'} `do [[${region##*/} = ~ ^\. ] & & continue [[`hadoop fs-du-s $region | awk {'print $1'} `- gt $SIZE]] & & split ${region##*/} done # check after split sleep 60fifor region in `hadoop fs-ls / hbase/data/default/$TABLE | awk {' print $8'} `do [[${region##*/} = ~ ^\. ] & & continue [[`hadoop fs-du-s $region | awk {'print $1'} `- gt $SIZE]] & & echo "${region##*/} (`hadoop fs-du-h $region | awk {' print $1 $2'}`) is a huge region" | | echo "${region##*/} (`hadoop fs-du-s-h $region | awk {'print $1 $2'} `) is a small region" done

Hbase-scan.py

Import subprocessimport datetimeimport argparseimport csvimport gzipimport happybaseimport loggingdef connect_to_hbase (): return happybase.Connection ('itr-hbasetest01') def main (): logging.basicConfig (format='% (asctime) s% (name) s% (levelname) s:% (message) argp = argparse.ArgumentParser (description='EventLog Reader') argp.add_argument Dest='prefix') argp.add_argument ("scan start") argp.add_argument ("row_prefix=args.prefix, batch_size=1000, limit=int (args.limit)" (row_prefix=args.prefix, batch_size=1000, limit=int (args.limit)) args = argp.parse_args () hbase_conn = connect_to_hbase () table = hbase_conn.table (args.table) logging.info ("scan start") Filter=args.filter) logging.info ("scan done") I = 0 for key, data in scanner: logging.info (key) print key iTunes 1 logging.info ('% s rows read in total', i) if _ _ name__ = ='_ main__': main ()