HCM

Happy Coding Monkey

Linux Cheat Sheet

2021-05-01


General

zip -Z store -r  myfolder.zip myfolder
# decompress filename.tar.xx
tar xvf archive.tar.xz
# GNU tar recognizes the format by itself!

Bash

Inside single quotes everything is preserved literally, without exception.

format date time

date=$(date '+%Y%m%d_%H%M%S')

extract value from file content, and use in shell script.

SQL_TEXT="$(IFS=''; cat ~/dump/tbl1.sql)"
mysql -h 127.0.0.1 -e "${SQL_TEXT}"

get the directory in which a Bash script is located

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"

copy while skipping .git folder by default.

cp GIT_REPO/*  target_dir/

copy only when the SOURCE file is newer than the destination file or when the destination file is missing

cp -u -r /home/peter/dst/* .

find .go file except the vendor directory

find . -path ./vendor -prune -false -o -name '*.go'`

find executable binary in a directory

find /usr/local/bin -type f -executable -exec sh -c 'test "$(head -c 2 "$1")" != "#!"' sh {} \; -print

grant access to directories recursively

find . -type d  -exec  chmod +x {} \;

find and grep

find . -name '*_test.go' -exec grep -Hn 127.0.0.1 {} \;

find . -name '*_test.go' -exec grep -Hn [golang.org/x](http://golang.org/x)  {}\;

find . -name '*.go' -exec grep --color -Hn "TODO:" {} \;

find . -name '*.[ch]' -exec grep --color -Hn 'struct page {' {} \;

grep "\.[ch]$" source_index.txt

## find TODO in code repo, ignore the binary files
grep --binary-files=without-match -nr "TODO:" .

grep with regular expression

## find boost:: and std:: usage in code base
## extract value using grep
find .  -name "*.[hc]*" |xargs grep -Poh "\Kstd::\w+" {}  |sort |uniq -c |sort -h > ~/tmp/cpp_std.txt

find .  -name "*.[hc]*" |xargs grep -Poh "\Kboost::\w+" {}  |sort |uniq -c |sort -h > ~/tmp/cpp_boost.txt

grep:
    -o Print only the matched (non-empty) parts of a matching line, with each such part on a separate output line.
	-h Suppress the prefixing of file names on output.

	## mysql
    109 std::list
    110 std::cerr
    110 std::make_pair
    149 std::min
    151 std::nothrow
    181 std::swap
    202 std::map
    226 std::pair
    228 std::ostream
    251 std::endl
    341 std::size_t
    529 std::cout
   1008 std::vector
   3723 std::string
   ...
     50 boost::range_iterator
     90 boost::end
     93 boost::begin
    248 boost::geometry

list sub-directories by size

du -h --max-depth=1 | sort -h

cloc: code summary

cloc  --not-match-f=".*_test.go" src/database/sql
Language                     files          blank        comment           code
-------------------------------------------------------------------------------
Go                               5            500           1472           3014


cloc  --match-f=".*_test.go" src/database/sql
Language                     files          blank        comment           code
-------------------------------------------------------------------------------
Go                               7            848            449           5721

find the top 10 largest file in directory recursively.

find . -type f -printf "%s\t%p\n" | sort -rn | head -10

find out binaries provided by a package.

dpkg -L mysql-community-server |grep -E '/s?bin/'

for loops

## curl 100 times, and write the results line by line

for i in {1..100}; do curl -s http://34.69.165.177/; echo '' ;done > result.txt

for f in *.sql; do (cat "${f}"; echo) >> all.txt; done

for dt in 2022-07-05 2022-07-06 2022-07-07 2022-07-08 2022-07-09 2022-07-10; do echo $dt; done

directory checksum

find somedir -type f -exec md5sum {} \; | sort -k 2 | md5sum

xargs

aws dynamodb list-tables | jq '.TableNames[]' > /tmp/dynamo_tables_test.txt

## xargs and dump files with parameterized file name
cat /tmp/dynamo_tables_test.txt | xargs -I{} sh -c 'aws dynamodb update-table --table-name="$1" --billing-mode PAY_PER_REQUEST > "$1.json"' -- {}

sed

## parse environment variables
sed -z 's/$/\n/' /proc/4185/environ |grep -a GO
cat /proc/1/environ |tr '\0' '\n'
cat /proc/1/cmdline |tr '\0' '\n'
strings /proc/1/environ

## filter lines by keyword(/struct/!d) and keep the second line(2p)
sudo cat /proc/slabinfo |sed -e '2p' -e '/struct/!d'

extract substring with regex using sed

redis-cli -h 127.0.0.1 -p 6379 client list >/tmp/clients.txt
# id=474958 addr=10.0.66.226:46704 fd=14 name= age=554798 idle=1 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=rcmd=ping
# id=474959 addr=10.0.66.226:46699 fd=23 name= age=554798 idle=1 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=rcmd=ping

awk '{print $2}' /tmp/clients.txt | sed -n 's/addr=\([0-9.]*\):.*/\1/p' | sort |uniq -c
 2 10.0.66.226

field select using awk

$ netstat -anp |grep  chia |grep LISTEN |awk '{ print $7 }' |sort |uniq | awk 'BEGIN { FS = "/" } { print $2}'
chia_daemon
chia_wallet
chia_full_nod
chia_farmer
chia_harveste

filter field by value using awk

lsmod | awk '{if ($3 != 0) print $1, $2, $3, $4}'

# filter mysql slow query log
grep Rows_examined mysql-slowquery.log |awk '{ print $9}' |sort -h |less

8 Powerful Awk Built-in Variables – FS, OFS, RS, ORS, NR, NF, FILENAME, FNR

Find the largest file in a directory and its subdirectories

find . -printf '%s %p\n'| sort -nr | head -10

shell parameter expansion

Inside single quotes everything is preserved literally, without exception.

That means you have to close the quotes, insert something, and then re-enter again.

'before'"$variable"'after'
'before'"'"'after'
'before'\''after'

replace string in bash

# ${parameter/pattern/string}
# (https://www.gnu.org/software/bash/manual/bash.html#Shell-Parameter-Expansion)
# If pattern begins with ‘/’, all matches of pattern are replaced with string.
IP=10.0.2.15

echo ${IP/\./_}
10_0.2.15

echo ${IP//\./_}
10_0_2_15

batch rename file

for i in ./docdb_* ; do mv "$i" "${i/blog_q2/p2p_q1}" ; done
for i in ./*.mp3 ; do mv "$i" "${i/ /_}" ; done

patch file

diff -u working/slang.c latest/slang.c > 1.patch
patch -u prod/other.c -i 1.patch

disable bash history

## turn off bash history:
unset HISTFILE
#or
set +o history

## turn on bash history
set -o history

log and standard output

docker logs aaasdas9c356 > docker.log 2>&1

journalctl --since "2020-01-02 05:17"

## system log 15 min before. (in `/var/logs/messages` before)
journalctl -k --since "10min ago”

## watch logs in directory
## ignored lines started with `==>` and empty lines
tail -f ./log | grep -v --line-buffered -e "^==> . <==$" | grep -v -e '^$'

Debug

reboot record

last reboot | less

measure command executing time

da=$(date +'%s.%3N'); XXX; db=$(date +'%s.%3N'); echo "$(echo "($db - $da)"|bc)"

while [ 1 = 1 ];do da=$(date +'%s.%3N');mysql --defaults-file=./my.cnf -D windmills_large -e "insert into windmills_test select null,uuid,millid,kwatts_s,date,location,active,time,strrecordtype from windmills4 limit 1;" -e "select count(*) from windmills_large.windmills_test;" > /dev/null;db=$(date +'%s.%3N'); echo "$(echo "($db - $da)"|bc)";sleep 1;done

Networking

one line http file server in Python

$ python2 -m SimpleHTTPServer 8000
$ python3 -m http.server

DNS

# dig using specified DNS server
dig @8.8.8.8 tinyurl.com

# clean DNS cache
sudo systemd-resolve --flush-caches

lsof

$ sudo lsof -nP -i4TCP:1080 -sTCP:LISTEN
$ sudo lsof -nP -i4TCP:1080 -sTCP:ESTABLISHED
$ sudo lsof -iTCP -sTCP:LISTEN -n -P

# -a means AND in lsof
$ sudo lsof -itcp -a -p "1280595"

tcpdump

$ sudo tcpdump -i all host 127.0.0.1 and port 1080 -w cap1.pcap

ip

# Bring wlan0 online
ip link set wlan0 up

# Bring wlan0 offline
ip link set wlan0 down

check NIC speed

cat /sys/class/net/enp4s0/speed
ethtool enp4s0

iperf

# server
iperf3 -s

# client
iperf3 -c 192.168.2.1

ssh tunnel

# listen on the remote side, and forward connection to the local side
ssh -p 22 -qngfNTR 8082:localhost:8080 dev2

check linux Ephemeral port range

cat /proc/sys/net/ipv4/ip_local_port_range

ssh and execute remote commands

Q: error: Pseudo-terminal will not be allocated because stdin is not a terminal A: https://stackoverflow.com/questions/7114990/pseudo-terminal-will-not-be-allocated-because-stdin-is-not-a-terminal

Q: escape single quotes with in string A: https://stackoverflow.com/questions/1250079/how-to-escape-single-quotes-within-single-quoted-strings

ssh -tt -i  ~/.ssh/ssh1.pem user1@JUMP-SERVER \
'ssh -tt -i ssh2.pem  user2@INTERNAL-SERVER '"'"'tmux -L session1  attach'"'"''

tcp ping with hping3

> hping -S -p 80 www.sunet.se
HPING www.sunet.se (eth0 192.36.171.155): S set, 40 headers + 0 data bytes
len=46 ip=192.36.171.155 ttl=59 DF id=0 sport=80 flags=SA seq=0 win=5840 rtt=0.7 ms

nnamp (network scanning)

nmap -sV 192.168.28.0/24 | grep -wE '(scan report|ssh)'

## check open port of host
nmap 192.168.28.14

System Admin

extend swap

$ swapon --show
NAME       TYPE SIZE USED PRIO
/.swapfile file   4G 1.5G   -2

$ ll /.swapfile
-rw------- 1 root root 4294967296 Aug 18  2022 /.swapfile


$ sudo fallocate -l 32G /swapfile
$ sudo mkswap /swapfile
$ sudo swapon /swapfile

$ swapon --show
NAME       TYPE SIZE USED PRIO
/.swapfile file   4G 1.5G   -2
/swapfile  file  32G   0B   -3

File system

mount local image file to file system

$ mount -o loop,offset=63963136 RetroPie_2017_8_29_rom.img /mnt/2

# find offset if multiple partition exists.
$ parted RetroPie_2017_8_29_rom.img
Disk /home/pi/RetroPie_2017_8_29_rom.img: 3276MB
Sector size (logical/physical): 512B/512B
Partition Table: msdos
Disk Flags:

Number  Start   End     Size    Type     File system  Flags
 1      4194kB  64.0MB  59.8MB  primary  fat16        lba
 2      64.0MB  3275MB  3211MB  primary  ext4

(parted) u
Unit?  [compact]? B
(parted) print
Model:  (file)
Disk /home/pi/RetroPie_2017_8_29_rom.img: 3275999744B
Sector size (logical/physical): 512B/512B
Partition Table: msdos
Disk Flags:

Number  Start      End          Size         Type     File system  Flags
 1      4194304B   63963135B    59768832B    primary  fat16        lba
 2      63963136B  3274702847B  3210739712B  primary  ext4

mount external devices, preventing errors being reported at boot. link.

/dev/xvdh1 /myfs xfs defaults,nofail,x-systemd.device-timeout=30 0 0

check device file system df -T, lsblk -no name, fstype, fdisk -l, fdisk -x, blkid /dev/nvme1n1, mount

Package

install with dependency

yum install ./percona-toolkit-3.3.1-1.el7.x86_64.rpm

install libs

library missing error when compiling mosh

checking for protobuf... no
configure: error: Package requirements (protobuf) were not met:

No package 'protobuf' found

Consider adjusting the PKG_CONFIG_PATH environment variable if you
installed software in a non-standard prefix.

Alternatively, you may set the environment variables protobuf_CFLAGS
and protobuf_LIBS to avoid the need to call pkg-config.
See the pkg-config man page for more details.

search and install

sudo apt install pkg-config

sudo apt search protobuf
sudo apt install libprotobuf-dev

install pkg in alpinelinux container:

## useful to debug connection in Pod
apk add lsof bind-tools

apk search xxx # support re
apk add hping3 --update-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing

list all binaries from $PATH

compgen -c

list the binaries installed by a Debian package

binaries () { dpkg -L "$1" | grep -Po '.*/bin/\K.*'; }

$ binaries gcc-riscv64-linux-gnu
riscv64-linux-gnu-gcc
riscv64-linux-gnu-gcc-ar
riscv64-linux-gnu-gcc-nm
riscv64-linux-gnu-gcc-ranlib
riscv64-linux-gnu-gcov
riscv64-linux-gnu-gcov-dump
riscv64-linux-gnu-gcov-tool

goreman to start multiple process.

Bootloader GRUB

set resolution in GRUB boot screen

## edit /etc/default/grub
GRUB_GFXMODE=1024x768x32
GRUB_GFXPAYLOAD_LINUX=keep
GRUB_DEFAULT="2"

## sudo update-grub2

Other

copy to clipboard of X window

cat ~/.ssh/id_rsa.pub | xclip -selection c

TTS (Text-to-Speech)

spd-say -t female3 "Good morning, Jimmy”

vim

edit by vim, not using config

vim -u NORC xxx

vim hex mode: %! xxd

atop

atop: memory accumulated by process name