Linux Cheat Sheet
2021-05-01
General
- zip/tar directory without compressing
zip -Z store -r myfolder.zip myfolder
- decompress
# decompress filename.tar.xx
tar xvf archive.tar.xz
# GNU tar recognizes the format by itself!
Bash
Inside single quotes everything is preserved literally, without exception.
format date time
date=$(date '+%Y%m%d_%H%M%S')
extract value from file content, and use in shell script.
SQL_TEXT="$(IFS=''; cat ~/dump/tbl1.sql)"
mysql -h 127.0.0.1 -e "${SQL_TEXT}"
get the directory in which a Bash script is located
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
copy while skipping .git folder by default.
cp GIT_REPO/* target_dir/
copy only when the SOURCE file is newer than the destination file or when the destination file is missing
cp -u -r /home/peter/dst/* .
find .go file except the vendor directory
find . -path ./vendor -prune -false -o -name '*.go'`
find executable binary in a directory
find /usr/local/bin -type f -executable -exec sh -c 'test "$(head -c 2 "$1")" != "#!"' sh {} \; -print
grant access to directories recursively
find . -type d -exec chmod +x {} \;
find and grep
find . -name '*_test.go' -exec grep -Hn 127.0.0.1 {} \;
find . -name '*_test.go' -exec grep -Hn [golang.org/x](http://golang.org/x) {}\;
find . -name '*.go' -exec grep --color -Hn "TODO:" {} \;
find . -name '*.[ch]' -exec grep --color -Hn 'struct page {' {} \;
grep "\.[ch]$" source_index.txt
## find TODO in code repo, ignore the binary files
grep --binary-files=without-match -nr "TODO:" .
grep with regular expression
## find boost:: and std:: usage in code base
## extract value using grep
find . -name "*.[hc]*" |xargs grep -Poh "\Kstd::\w+" {} |sort |uniq -c |sort -h > ~/tmp/cpp_std.txt
find . -name "*.[hc]*" |xargs grep -Poh "\Kboost::\w+" {} |sort |uniq -c |sort -h > ~/tmp/cpp_boost.txt
grep:
-o Print only the matched (non-empty) parts of a matching line, with each such part on a separate output line.
-h Suppress the prefixing of file names on output.
## mysql
109 std::list
110 std::cerr
110 std::make_pair
149 std::min
151 std::nothrow
181 std::swap
202 std::map
226 std::pair
228 std::ostream
251 std::endl
341 std::size_t
529 std::cout
1008 std::vector
3723 std::string
...
50 boost::range_iterator
90 boost::end
93 boost::begin
248 boost::geometry
list sub-directories by size
du -h --max-depth=1 | sort -h
cloc: code summary
cloc --not-match-f=".*_test.go" src/database/sql
Language files blank comment code
-------------------------------------------------------------------------------
Go 5 500 1472 3014
cloc --match-f=".*_test.go" src/database/sql
Language files blank comment code
-------------------------------------------------------------------------------
Go 7 848 449 5721
find the top 10 largest file in directory recursively.
find . -type f -printf "%s\t%p\n" | sort -rn | head -10
find out binaries provided by a package.
dpkg -L mysql-community-server |grep -E '/s?bin/'
for loops
## curl 100 times, and write the results line by line
for i in {1..100}; do curl -s http://34.69.165.177/; echo '' ;done > result.txt
for f in *.sql; do (cat "${f}"; echo) >> all.txt; done
for dt in 2022-07-05 2022-07-06 2022-07-07 2022-07-08 2022-07-09 2022-07-10; do echo $dt; done
directory checksum
find somedir -type f -exec md5sum {} \; | sort -k 2 | md5sum
xargs
aws dynamodb list-tables | jq '.TableNames[]' > /tmp/dynamo_tables_test.txt
## xargs and dump files with parameterized file name
cat /tmp/dynamo_tables_test.txt | xargs -I{} sh -c 'aws dynamodb update-table --table-name="$1" --billing-mode PAY_PER_REQUEST > "$1.json"' -- {}
sed
## parse environment variables
sed -z 's/$/\n/' /proc/4185/environ |grep -a GO
cat /proc/1/environ |tr '\0' '\n'
cat /proc/1/cmdline |tr '\0' '\n'
strings /proc/1/environ
## filter lines by keyword(/struct/!d) and keep the second line(2p)
sudo cat /proc/slabinfo |sed -e '2p' -e '/struct/!d'
extract substring with regex using sed
redis-cli -h 127.0.0.1 -p 6379 client list >/tmp/clients.txt
# id=474958 addr=10.0.66.226:46704 fd=14 name= age=554798 idle=1 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=rcmd=ping
# id=474959 addr=10.0.66.226:46699 fd=23 name= age=554798 idle=1 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=rcmd=ping
awk '{print $2}' /tmp/clients.txt | sed -n 's/addr=\([0-9.]*\):.*/\1/p' | sort |uniq -c
2 10.0.66.226
field select using awk
$ netstat -anp |grep chia |grep LISTEN |awk '{ print $7 }' |sort |uniq | awk 'BEGIN { FS = "/" } { print $2}'
chia_daemon
chia_wallet
chia_full_nod
chia_farmer
chia_harveste
filter field by value using awk
lsmod | awk '{if ($3 != 0) print $1, $2, $3, $4}'
# filter mysql slow query log
grep Rows_examined mysql-slowquery.log |awk '{ print $9}' |sort -h |less
8 Powerful Awk Built-in Variables – FS, OFS, RS, ORS, NR, NF, FILENAME, FNR
Find the largest file in a directory and its subdirectories
find . -printf '%s %p\n'| sort -nr | head -10
shell parameter expansion
Inside single quotes everything is preserved literally, without exception.
That means you have to close the quotes, insert something, and then re-enter again.
'before'"$variable"'after'
'before'"'"'after'
'before'\''after'
replace string in bash
# ${parameter/pattern/string}
# (https://www.gnu.org/software/bash/manual/bash.html#Shell-Parameter-Expansion)
# If pattern begins with ‘/’, all matches of pattern are replaced with string.
IP=10.0.2.15
echo ${IP/\./_}
10_0.2.15
echo ${IP//\./_}
10_0_2_15
batch rename file
for i in ./docdb_* ; do mv "$i" "${i/blog_q2/p2p_q1}" ; done
for i in ./*.mp3 ; do mv "$i" "${i/ /_}" ; done
patch file
diff -u working/slang.c latest/slang.c > 1.patch
patch -u prod/other.c -i 1.patch
disable bash history
## turn off bash history:
unset HISTFILE
#or
set +o history
## turn on bash history
set -o history
log and standard output
docker logs aaasdas9c356 > docker.log 2>&1
journalctl --since "2020-01-02 05:17"
## system log 15 min before. (in `/var/logs/messages` before)
journalctl -k --since "10min ago”
## watch logs in directory
## ignored lines started with `==>` and empty lines
tail -f ./log | grep -v --line-buffered -e "^==> . <==$" | grep -v -e '^$'
Debug
reboot record
last reboot | less
measure command executing time
da=$(date +'%s.%3N'); XXX; db=$(date +'%s.%3N'); echo "$(echo "($db - $da)"|bc)"
while [ 1 = 1 ];do da=$(date +'%s.%3N');mysql --defaults-file=./my.cnf -D windmills_large -e "insert into windmills_test select null,uuid,millid,kwatts_s,date,location,active,time,strrecordtype from windmills4 limit 1;" -e "select count(*) from windmills_large.windmills_test;" > /dev/null;db=$(date +'%s.%3N'); echo "$(echo "($db - $da)"|bc)";sleep 1;done
Networking
one line http file server in Python
$ python2 -m SimpleHTTPServer 8000
$ python3 -m http.server
DNS
# dig using specified DNS server
dig @8.8.8.8 tinyurl.com
# clean DNS cache
sudo systemd-resolve --flush-caches
lsof
$ sudo lsof -nP -i4TCP:1080 -sTCP:LISTEN
$ sudo lsof -nP -i4TCP:1080 -sTCP:ESTABLISHED
$ sudo lsof -iTCP -sTCP:LISTEN -n -P
# -a means AND in lsof
$ sudo lsof -itcp -a -p "1280595"
tcpdump
$ sudo tcpdump -i all host 127.0.0.1 and port 1080 -w cap1.pcap
ip
# Bring wlan0 online
ip link set wlan0 up
# Bring wlan0 offline
ip link set wlan0 down
check NIC speed
cat /sys/class/net/enp4s0/speed
ethtool enp4s0
iperf
# server
iperf3 -s
# client
iperf3 -c 192.168.2.1
ssh tunnel
# listen on the remote side, and forward connection to the local side
ssh -p 22 -qngfNTR 8082:localhost:8080 dev2
check linux Ephemeral port range
cat /proc/sys/net/ipv4/ip_local_port_range
ssh and execute remote commands
Q: error: Pseudo-terminal will not be allocated because stdin is not a terminal A: https://stackoverflow.com/questions/7114990/pseudo-terminal-will-not-be-allocated-because-stdin-is-not-a-terminal
Q: escape single quotes with in string A: https://stackoverflow.com/questions/1250079/how-to-escape-single-quotes-within-single-quoted-strings
ssh -tt -i ~/.ssh/ssh1.pem user1@JUMP-SERVER \
'ssh -tt -i ssh2.pem user2@INTERNAL-SERVER '"'"'tmux -L session1 attach'"'"''
tcp ping with hping3
> hping -S -p 80 www.sunet.se
HPING www.sunet.se (eth0 192.36.171.155): S set, 40 headers + 0 data bytes
len=46 ip=192.36.171.155 ttl=59 DF id=0 sport=80 flags=SA seq=0 win=5840 rtt=0.7 ms
nnamp (network scanning)
nmap -sV 192.168.28.0/24 | grep -wE '(scan report|ssh)'
## check open port of host
nmap 192.168.28.14
System Admin
extend swap
$ swapon --show
NAME TYPE SIZE USED PRIO
/.swapfile file 4G 1.5G -2
$ ll /.swapfile
-rw------- 1 root root 4294967296 Aug 18 2022 /.swapfile
$ sudo fallocate -l 32G /swapfile
$ sudo mkswap /swapfile
$ sudo swapon /swapfile
$ swapon --show
NAME TYPE SIZE USED PRIO
/.swapfile file 4G 1.5G -2
/swapfile file 32G 0B -3
File system
mount local image file to file system
$ mount -o loop,offset=63963136 RetroPie_2017_8_29_rom.img /mnt/2
# find offset if multiple partition exists.
$ parted RetroPie_2017_8_29_rom.img
Disk /home/pi/RetroPie_2017_8_29_rom.img: 3276MB
Sector size (logical/physical): 512B/512B
Partition Table: msdos
Disk Flags:
Number Start End Size Type File system Flags
1 4194kB 64.0MB 59.8MB primary fat16 lba
2 64.0MB 3275MB 3211MB primary ext4
(parted) u
Unit? [compact]? B
(parted) print
Model: (file)
Disk /home/pi/RetroPie_2017_8_29_rom.img: 3275999744B
Sector size (logical/physical): 512B/512B
Partition Table: msdos
Disk Flags:
Number Start End Size Type File system Flags
1 4194304B 63963135B 59768832B primary fat16 lba
2 63963136B 3274702847B 3210739712B primary ext4
mount external devices, preventing errors being reported at boot. link.
/dev/xvdh1 /myfs xfs defaults,nofail,x-systemd.device-timeout=30 0 0
check device file system
df -T, lsblk -no name, fstype, fdisk -l, fdisk -x, blkid /dev/nvme1n1, mount
Package
install with dependency
yum install ./percona-toolkit-3.3.1-1.el7.x86_64.rpm
install libs
library missing error when compiling mosh
checking for protobuf... no
configure: error: Package requirements (protobuf) were not met:
No package 'protobuf' found
Consider adjusting the PKG_CONFIG_PATH environment variable if you
installed software in a non-standard prefix.
Alternatively, you may set the environment variables protobuf_CFLAGS
and protobuf_LIBS to avoid the need to call pkg-config.
See the pkg-config man page for more details.
search and install
sudo apt install pkg-config
sudo apt search protobuf
sudo apt install libprotobuf-dev
install pkg in alpinelinux container:
## useful to debug connection in Pod
apk add lsof bind-tools
apk search xxx # support re
apk add hping3 --update-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing
list all binaries from $PATH
compgen -c
list the binaries installed by a Debian package
binaries () { dpkg -L "$1" | grep -Po '.*/bin/\K.*'; }
$ binaries gcc-riscv64-linux-gnu
riscv64-linux-gnu-gcc
riscv64-linux-gnu-gcc-ar
riscv64-linux-gnu-gcc-nm
riscv64-linux-gnu-gcc-ranlib
riscv64-linux-gnu-gcov
riscv64-linux-gnu-gcov-dump
riscv64-linux-gnu-gcov-tool
goreman to start multiple process.
Bootloader GRUB
set resolution in GRUB boot screen
## edit /etc/default/grub
GRUB_GFXMODE=1024x768x32
GRUB_GFXPAYLOAD_LINUX=keep
GRUB_DEFAULT="2"
## sudo update-grub2
Other
copy to clipboard of X window
cat ~/.ssh/id_rsa.pub | xclip -selection c
TTS (Text-to-Speech)
spd-say -t female3 "Good morning, Jimmy”
vim
edit by vim, not using config
vim -u NORC xxx
vim hex mode: %! xxd
atop
atop: memory accumulated by process name