Linux Tips
Various Linux commands and howto
regex - match - replace - Vim
VIM - Find & Replace
:%s/foo/bar/g # find and replace all foo with bar
:s/foo/var/g # only the first one..current line
regexp | grep
Grep blank lines: `^(?:[\t ]*(?:\r?\n|\r))+`
Grep ip: `grep -Eo '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}'`
Sed
Match & Replace
sed 's/${pattern_to_match}/${new_pattern}/g'
echo "ansible is orchestration tool" | sed 's/orchestration/automation/g'
ansible is automation tool
# Remove last character
echo "toto" | sed 's/.$//g' # => tot
# Remove first one
echo "toto" | sed 's/^.//g' # => oto
# Remove the both
echo "BtotoE" | sed 's/^.//;s/.$//'
# Remove "" around 25320
echo '"buildDuration" : "25320",' | sed 's/"\([0-9][0-9]*\)",/\1/g'
# backslash option: `\1`
jq "._source"|sed 's/^\([ ]*"buildDuration": \)"\([0-9][0-9]*\)"\(,\)$/\1\2\3/g'
find
find
find folder
find ./ -type d -name "opt"
find ${path} -type f -name "myfile"
awk
awk to display/get ...
awk '{print $1}' ${file} # first column
awk '{print $NF}' ${file} # last one
awk '{print $1 $3}' ${used} #1st and 3th
# with docker ps to get only container ID column
docker ps | awk '{print $1}'
CONTAINER
477046ec9337
ca56bfac6e7b
# Possible to specify a sperator, example "//"
echo "https://doc.enoks.fr" | awk -F// '{print $2}'
doc.enoks.fr
echo "https://doc.enoks.fr" | awk -F// '{print $1}'
https:
cut - trim - uppercase
tr -d ' ' to trim, remove blank space
To split: cut or awk or sed
Example
#reduce space in one and then split with
# delimer ' '(one space) and return 2nd value
echo "image: lxabcd.com:500" | tr -s ' ' |cut -s -d ' ' -f2
lxabcd.com:500
# Convert to uppercase
echo 'tata' | tr a-z A-Z # => TATA
name="toto", echo ${name^}=Toto echo ${name^^}=TOTO
crontab
Manage
crontab -l # read
crontab -e # write
crontab -l -u toto # to display user toto crons
# A help to generate your cron entry frequency: `https://crontab.guru/`
# It is also possible to put a scripts in `cron.hourly`, `cron.daily`, `cron.monthly`
# Script name should be without extension: **mysrcript** and not ~myscript.sh~
sudo run-parts --list /etc/cron.daily
sudo run-parts --test /etc/cron.daily
sudo run-parts /etc/cron.daily # to force daily cron execution
journalctl
View logs
journalctl -u ${service} --since "2019-03-25 04:30:00"
journalctl --since "2016-02-10 21:00:00" --until "2016-02-10 22:00:00"
logrotate
Check if logrotate is working
cat /var/lib/logrotate/status # to see execution logs
logrotate -vf /etc/logrotate.d # Force execution
rsync
Sync
rsync_options="-rlptogD -vz --delete-after --exclude-from=fichier"
rsync $rsync_options -e "ssh -i /path/to/ssh/key" $rsync_destinationpath $rsync_sourcepath
Data Backup example
ssh-keygen
Generate/Create a public SSH key from the private SSH key
ssh-keygen -y -f id_rsa will output the public key. You can save it in the file.
reload current shell
exec -l $SHELL
chage - Manage password expiration
chage
chage -l username ###pour voir les infos concernant
sudo chage username #pour modifier les dates d'expiration
**Minimum Password Age to 0
Maximum Password Age to 99999
les autres juste taper entrer
ou directement chage -I -1 -m 0 -M 99999 -E -1 username
lsattr to prevent file deletion
To prevent file deletion
lsattr # like ls, if ---i--- mean deletion is blocked
# prevent
chattr +i file
chattr +i -R folder/
# allow
sudo chattr -i ....
lsattr #pour le ls -al
seq - loop
To generate a list with range, by completion
seq -f "51.161.107.%g" 168 171
51.161.107.168
51.161.107.169
51.161.107.170
51.161.107.171
seq 1 7
1
3
for elt in `seq 8 11`; do echo element: $elt; done
element: 8
element: 1
for elt in {11..13}; do echo element: $elt; done
element: 11
element: 12
element: 13
# -w to have 2 digit: 00 01
for i in `seq -w 0 13`; do echo $i; done
00
01
.
13
sudo systemctl daemon-reload
for p in $(seq 9200 9210); do sudo systemctl enable iperf3-server@$p ; done # enable 10 iperf3 daemon from 9200 9210
systemctl status iperf3-server@*
list - loop - while/for
whileOR for?
# With while
cat toto.txt | while read -r line; do echo $line; done
# With for
for elt in ${tabofpath[@]}
do
echo $elt
done
# create a table with a list result from delimiter(ex: ,)
IFS=',' read -r -a array <<< "$cluster"
for node in ${array[@]}
do
lvextend - lvreduce - resize2fs
Manage
lvextend -L+1G /dev/rootvg/var_lv #LV de /var
resize2fs /dev/mapper/rootvg-var_lv # path in /etc/fstab
vgextend levg lepv
vgreduce levg lepv
Fix filesystem /etc/fstab
To fix issue with mounted fs, volume
# Update /etc/fstab as it should be, then
fsck -Af -M
archive - tar - zip
To compress/decompress, manage archive
tar cvf archive.tar file1 file2
tar cvf folder.tar folder/
tar zcvf folder.tar.gz folder # tar.gz
# unzip, decompress
tar xvf archive.tar
tar -zxvf doc.tar.gz
There is also the command zip and unzip
# Useful if you want to easily encript the archive
-e # Interactive encription
zip toto.zip toto.txt tata.txt -e
- P # Non interactive encryption
zip toto.zip toto.txt tata.txt
zip toto.zip -P $password # to encrypt toto.zip
unzip folder.zip -d ${destination_path}
terminal Terminator
- Install the package :
apt-get install terminator - Config will be stored in :
.config/terminator/config - Doc :
https://doc.ubuntu-fr.org/terminator
shortcuts
* Ctrl-Shift-E : Scinder la fenêtre verticalement.
* Ctrl-Shift-O : Scinder la fenêtre horizontalement.
* Ctrl-Shift-W : Fermer la fenêtre courante.
* Ctrl-Shift-X : Agrandir la fenêtre courante.
* Ctrl-Tab : Changer de "split" de fenêtre
* Alt+A Broadcast to All terminals.
* Alt+O Broadcast Off.
* zoomer terminal: ctrl shift +++
* zoom normal: ctrl shift zero
Bash Script argument with flag --
Example of how to write Bash script to take arguments with -- option
Arguments with flag
# Expect 2 mandatory variables: clustername & namespace AND 1 optional: validity
usage() {
echo "Usage: $0 --clustername <postgres_cluster_name> --namespace <target_namespace> \
[--validity <days|default 365>]"
printf "\n\nDescription: This script will generate a CA certificate \
and then create and sign a certificate for a user streaming_replica\n"
exit 1
}
# Set default values
validity=365
# Check if the number of arguments is correct
if [ "$#" -lt 4 ] || [ "$#" -gt 6 ]; then
usage
fi
# Parse command line arguments
while [ "$#" -gt 0 ]; do
case "$1" in
--clustername)
clustername="$2"
shift 2
;;
--namespace)
namespace="$2"
shift 2
;;
--validity)
validity="$2"
shift 2
;;
*)
usage
;;
esac
done
# Check if required parameters are provided
if [ -z "${clustername}" ] || [ -z "${namespace}" ]; then
usage
fi
# Output the values of parameters
echo "Cluster Name: ${clustername}"
echo "Namespace: ${namespace}"
echo "Certificate Validity (days): ${validity}"
Date - Timestamp - Duration
View
timestamp=$(date -u +%s%N)
date= date -u "+%FT%T%2N%z"
# convert timestamp to date
date -u -d @"${timestamp}" "+%FT%T%2N%z"
//date en timestamp
date -u -d "$mydate" +%s
# duration
duration=$(( $timestamp1 - $timestamp2 ))
duration=$(( $(date -d $date1 +%s) - $(date -d $date2 +%s) ))
View
python3
# with datetime
>>> import datetime
>>> s="2021-08-25T12:31:20.000+0000"
>>> datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%S.%f%z").timestamp()
1629894680.0
>>> time.mktime(datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%S.%f%z").timetuple())
1629887480.0
>>> int(datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%S.%f%z").timestamp())
1629894680
# current time timestamp
datetime.datetime.timestamp(datetime.datetime.now())
#from timestamp to time/date
>>> datetime.datetime.fromtimestamp(1629993675.874264).isoformat()
'2021-08-26T18:01:15.874264'
View
python3
# with time
>>> import time
>>> ts = time.gmtime()
>>> print(time.strftime("%Y-%m-%d %H:%M:%S", ts))
2021-08-26 16:11:31
>>> print(time.strftime("%c", ts))
Thu Aug 26 16:11:31 2021
>>> print(time.strftime("%s", ts))
1629990691
>>> print(time.strftime("%x %X", ts))
08/26/21 16:11:31
locale - date
Configure
timedatectl set-timezone Europe/Paris
dpkg-reconfigure locales
locale
localectl list-locales
localectl set-locale fr_FR.utf8
lsof
Useful when you delete files but disk space is not free.
Sometimes file is stucked
file is stucked
lsof | grep 'deleted' # To get deleted stucked files..
# equivalent if you have not lsof command
ls -l /proc/*/fd/* 2>/dev/null | grep 'deleted'
fuser $file_name # to know which process use this file
basic credential - Apache/Nginx
htpasswd
apt install apache2-utils # to install it
htpasswd -n toto # to create password for toto with prompt
echo warn /critical with color
To print colored warn/critical message with Bash
echo -e "[\\e[1;94mINFO\\e[0m] $*"
echo -e "[\\e[1;93mWARN\\e[0m] $*"
echo -e "[\\e[1;91mERROR\\e[0m] $*"
function log_info() {
echo -e "[\\e[1;94mINFO\\e[0m] $*"
}
log_info "--- \\e[32mdeploy\\e[0m (env: \\e[33;1m${stage}\\e[0m)"
log_info "--- looking for Kubernetes scripts in directory: \\e[33;1m${K8S_SCRIPTS_DIR}\\e[0m"
log_info "--- appname: \\e[33;1m${appname}\\e[0m"
log_info "--- appname_ssc: \\e[33;1m${appname_ssc}\\e[0m"
log_info "--- stage: \\e[33;1m${stage}\\e[0m"
log_info "--- hostname: \\e[33;1m${hostname}\\e[0m"
selinux
To disable selinux
sestatus
su -c "setenforce 0" #temporarily
conf: /etc/selinux/config....
dd
Generate file with a specific size
dd if=/dev/urandom bs=1048576 count=10 of=./10Mb.dat
dd if=/dev/urandom bs=1048576 count=100 of=./100Mb.dat
dd if=/dev/urandom bs=1048576 count=1024 of=./1Gb.dat
dd if=/dev/urandom bs=10485760 count=1024 of=./10Gb.dat
Dual boot - Grub Recovery
When you have a dual boot (Ubuntu -Windows) and you removethe Ubuntu Partition
(to swicth back to single boot or accidently)
Your PC will not automatically boot on Windows at the next tsartup/reboot.
It will start with console grub recovery
To fix it and access to your Windows, you should have a Bootable Windows disk/usb
- Plug the disk/usb
- Restart the PC and access to boot from the installation usb/disk
- click Repair Your Computer.
- Next, select Troubleshoot > Advanced Options > Command Prompt. Here
we use the Bootrec.exe tool using the fixbbr command.
bootrec
# fix Master Boot Record
bootrec /fixmbr
# Don't worry if it fails
# In my case, it fails but the issue was fixed
bootrec /fixboot
bootrec /scanos
use php with Nginx
Configuration
apt install php7.3-fpm # or just php-fpm then /etc/php/*/fpm/php.ini to know the exact version
service php7.3-fpm status
vim /etc/php/*/fpm/php.ini
cgi.fix_pathinfo=0 # chnage the value to 1 like cgi.fix_pathinfo=1
## Then add this in your nginx site conf
location ~ \.php$ {
include snippets/fastcgi-php.conf;
fastcgi_pass unix:/run/php/php7.3-fpm.sock;
}
bc - calculator
Bash/Shell calculator is: bc
echo "scale=1; 10/3"|bc
3.3
echo "scale=2; 10/3"|bc
3.33
echo "89 *7842"|bc
697938
echo "19-3"|bc
16
# comparison tips: `awk 'BEGIN{if('$a'>'$b')print '$a'" is max";else print '$b'" is max"}'`
a=7
b=3
awk 'BEGIN{if('$a'>'$b')print '$a'" is max";else print '$b'" is max"}'
7 is max
smtp - sendmail
Configuration
https://doc.ubuntu-fr.org/ssmtp
sudo su
apt-get install ssmtp
cp ssmtp.conf backupssmtp.conf
*************####config de ssmtp.conf
#
# Config file for sSMTP sendmail
#
# The person who gets all mail for userids < 1000
# Make this empty to disable rewriting.
root=toto.tata@gmail.com
AuthUser=toto.tata@gmail.com
AuthPass=xxxxx
# The place where the mail goes. The actual machine name is required no
# MX records are consulted. Commonly mailhosts are named mail.domain.com
mailhub=smtp.gmail.com:587 mx-eu.equant.com:25
# Where will the mail seem to come from?
rewriteDomain=gmail.com orange.com
# The full hostname
# hostname -f
hostname=server.example.com
# Are users allowed to set their own From: address?
# YES - Allow the user to specify their own From: address
# NO - Use the system generated From: address
FromLineOverride=YES
UseSTARTTLS=YES
*************####config de revaliases
root:toto.tata@gmail.com:mail.fournisseur.com:587 root:toto@example.com:25
***************open port
sudo ufw allow out 587/tcp
sudo iptables -t filter -A INPUT -p tcp --sport 25 -j ACCEPT
sudo iptables -t filter -A OUTPUT -p tcp --dport 25 -j ACCEPTl
apt-get install mailutils
#https://myaccount.google.com/lesssecureapps?pli=1 //autorise les applications à se connecter
mail -s "email with bash" tintin@gmail.com