Linux
From Cheatsheet
Jump to navigationJump to search
This page is or will soon be an amalgamation of content from other pages
Basics
Ctrl + L = Clear the terminal-screen Ctrl + W = Delete word on the left Alt + D = Delete word on the right Ctrl + U = Delete everything left of the cursor Ctrl + K = Delete everything right of the cursor
< = Standard input is redirected > = Standard output is redirected 1> = Standard output is redirected 2> = Standard error is redirected >& = Standard output and error is redirected
Ctrl+R is used to reverse-search commands.
(reverse-i-search)`yum':
Common checks
Monitoring
# See CPU + RAM usage, system stats and open processes top # Only list processes making active use of the CPU top -i # Only list processes making active use of the CPU, and include the entire command being instead of just the tool-name top -ci # Prettier version of top that can be customized htop # Reimagined version of top, includes network and disk usage by default btop # Reimagined version of top that shows DISK READ and WRITE iotop # List all running processes ps aux
Systemd
# Open journalctl at the beginning journalctl -b # Open journalctl at the end journalctl -e # Open journalctl but include service information journalctl -x # Show journalctl logs for the sshd service, starting from the end journalctl -u sshd -e # Output contents directly to the shell journalctl --no-pager
OS & Distribution
# Print OS and host information hostnamectl # Show OS and distribution information cat /proc/version # Show OS and distribution information cat /etc/os-release # Print distribution-specific information lsb_release -a
Hardware & kernel
# List installed kernel modules lsmod # Print Kernel messages dmesg # Print Kernel messages with humanized timestamps dmesg -T # SCSI hardware information cat /proc/scsi/scsi # Print hardware/BIOS information dmidecode # Print hardware/BIOS information of a specific type dmidecode -t 1 # List all connected hardware lshw # List physical network hardware lshw -short -class network # List physical memory hardware lshw -class memory # Show PCI information lspci # Show verbose PCI information lspci -v # Show GPU info lshw -C display # List all block/filesystem devices lsblk # List block devices and partition tables fdisk -l
Pacemaker
# Show status of the pacemaker cluster pcs cluster status # Show status of the pacemaker service pcs status # Show configured pacemaker resources pcs resource config # Show a specific configured resource pcs resource show ResourceNameHere
Services
Common
systemctl
# List all services that are running or exited
systemctl
# List all services, running or otherwise
systemctl --all
# List all failed services
systemctl --state=failed
# Reset the failed service "nginx"
systemctl reset-failed nginx
# View the status of the "nfs-server" service
systemctl status nfs-server
# Output the config file of "rsyslog" to the shell
systemctl cat rsyslog
# Restart the "sshd" service, terminating established connections and re-parsing the configuration
systemctl restart sshd
# Reload the "nginx" service so that it only re-parses the configuration
systemctl reload nginx
# Stop the "nfs-ganesha" service so that it stops being run
systemctl stop nfs-ganesha
# Start the "nfs-ganesha" service so that it starts being run again
systemctl start nfs-ganesha
# Disable the "mariadb" service so that it doesn't start after the next boot
systemctl disable mariadb
# Enable the "mariadb" service so that it starts after the next boot.
systemctl enable mariadb
# Check the logs for all failed services
for i in $(systemctl --state=failed | head -n -4 | tail -n +2 | awk '{print $1}'); do systemctl --no-pager status "$i"; done
NTP
Timedatectl
# Show the current status of timedatectl timedatectl # List available timezones timedatectl list-timezones # Set the timezone to Amsterdam timedatectl set-timezone Europe/Amsterdam # Show verbose sync information timedatectl timesync-status
SNMP
V3 client installation
apt install snmpd snmp libsnmp-dev cp /etc/snmp/snmpd.conf /etc/snmp/snmpd.conf.bak systemctl stop snmpd net-snmp-create-v3-user -ro -X <CRYPTO-PASSWORD> -a SHA -X <PASSWORD> -x AES <USERNAME>
# /etc/snmp/snmpd.conf sysLocation NL;Zuid-Holland;Rotterdam, 78 MyStreet;2nd Floor;Server Room;Rack sysContact Me <me@example.org> agentaddress 192.168.0.10
systemctl start snmpd systemctl enable snmpd
# Test snmpwalk -v3 -a SHA -A "AUTHENTICATION PASSWORD" -x AES -X "CRYPTO PASSWORD" -l authPriv -u "MYUSER" localhost | head
CTDB
Checks
# Verify CTDB cluster status ctdb status # Show the allocated IP addresses and to which nodes they're bound ctdb ip # See the status of all CTDB-scripts ctdb scriptstatus ctdb event status # Show the time of the last failover the duration it took to recover ctdb uptime # See various statistics and data ctdb statistics # Use the onnode command to execute a command on all cluster nodes onnode all ctdb status
Commands
# Stop a ctdb cluster member ctdb stop # Start a stopped ctdb cluster member ctdb continue
Firewalls
UFW
Checks
# Show summary of UFW status ufw status # Show verbose UFW status ufw status verbose # Show UFW rules numbered ufw status numbered
Commands
# Allow access from a specific IP to a port and add a comment that show in the status ufw allow from 10.0.0.253 to any port 22 proto tcp comment 'Allow SSH access from XYZ location' # Delete numbered Firewall rule 56 ufw delete 56 # Disable UFW logging (prevent syslog spam) ufw logging off # Set UFW logging back to the default ufw logging low
Firewalld
SNMP access
# /etc/firewalld/services/snmp.xml <?xml version="1.0" encoding="utf-8"?> <service> <short>SNMP</short> <description>SNMP protocol</description> <port protocol="udp" port="161"/> </service>
firewall-cmd --reload firewall-cmd --zone=public --add-service snmp --permanent firewall-cmd --reload
Firewall-cmd
Checks
# List all available commands firewall-cmd -h # Check the configuration file of the firewall for errors firewall-cmd --check-config # Display the current state of firewall-cmd (running/shutdown) firewall-cmd --state # Display all available zones firewall-cmd --get-zones # List all whitelisted services firewall-cmd --list-services # List all services you can potentially enable firewall-cmd --get-services # List all added or enabled services and ports in more detail firewall-cmd --list-all # List verbose information for all zones firewall-cmd --list-all-zones # List verbose information for the public zone firewall-cmd --list-all --zone=public # See what port(s) are associated with the dns service firewall-cmd --info-service dns # List all opened ports firewall-cmd --list-ports # List kernel ruleset generated for nftables(?) nft list ruleset
Commands
# Reload the firewall firewall-cmd --reload # Whitelist the dns service, persistently even after reboot firewall-cmd --add-service=dns ; sudo firewall-cmd --runtime-to-permanent; firewall-cmd --reload # Whitelist the http service, persistently even after reboot firewall-cmd --add-service=http ; sudo firewall-cmd --runtime-to-permanent; firewall-cmd --reload # Remove the http service from the whitelist firewall-cmd --remove-service=http # Add port 1234 (tcp) to the whitelist firewall-cmd --add-port=1234/tcp # Remove port 1234 (tcp) from the whitelist firewall-cmd --remove-port=1234/tcp # Add port 2345 (udp) to the whitelist in zone external firewall-cmd --zone=external --add-port=2345/udp # Remove port 2345 (udp) from the whitelist for zone external firewall-cmd --zone=external --add-port=2345/udp # Add current configuration to configuration permanently firewall-cmd –runtime-to-permanent
DANGEROUS
# SHUT IT DOWN DOC - DROP ALL PACKETS AND EXPIRE EXISTING CONNECTIONS firewall-cmd --panic-on # ACCEPT PACKETS AGAIN firewall-cmd --panic-off
CSF
ConfigServer Security and Firewall
General
- Common configuration: /etc/csf/csf.conf
- Blacklist: /etc/csf/csf.deny
- Whitelist: /etc/csf/csf.allow
Installation
From the official instructions: https://download.configserver.com/csf/install.txt
Prerequisites
Perl Modules ============ While most should be installed on a standard perl installation the following may need to be installed manually: # On rpm based systems: yum install perl-libwww-perl.noarch perl-LWP-Protocol-https.noarch perl-GDGraph # On APT based systems: apt-get install libwww-perl liblwp-protocol-https-perl libgd-graph-perl
Install
cd /usr/src rm -fv csf.tgz wget https://download.configserver.com/csf.tgz tar -xzf csf.tgz cd csf sh install.sh # Next, test whether you have the required iptables modules: perl /usr/local/csf/bin/csftest.pl # Don't worry if you cannot run all the features, so long as the script doesn't report any FATAL errors
Checks
# Check the running status of csf csf status
Commands
# Commit config changes by restarting csf csf -r
csf.conf
Some common changes within the configuration file
# Set testing to 0 when your CSF configuration is 'production' ready TESTING = "0" # Allow access to any service you're hosting locally, for example https TCP_IN = "443" UDP_IN = "" # Allow all outwards HTTP/HTTPS traffic so you can yum/apt update TCP_OUT = "80,443" # Allow outgoing traceroute UDP_OUT = "33434:33523" # Allow your server to be pinged ICMP_IN = "0"
Formatting
The varying styles of formatting used in allow.conf
# Allow anything relating to the following IPs/ranges 192.168.10.0/24 # Our application breaks without this range 192.168.1.1 # Our gateway or something # Detailed entries based on Transport protocol, direction, Application protocol and IP tcp:in:d=22:s=7.7.7.7 # SSH access from our VPN udp:in:d=161:s=10.11.12.100 # SNMP Access tcp|in|d=22|s=fe80::1:/16 # IPV6 SSH access from our jumpgateway udp|in|d=3389|s=10.1.0.0/24 # RDP Access from our entire office range tcp|out|d=80,443|d=1.2.3.4/32 # Allow outgoing HTTP/HTTPS access via port 80 and 443 # Allow sending Syslog messages to our Syslog server udp|out|d=514|d=192.168.20.5 # UDP syslog server tcp|out|d=10514|d=192.168.20.5 # UDP syslog server # Allow sending queries to some DNS servers tcp|out|s=53|d=8.8.8.8 udp|out|s=53|d=1.1.1.1 udp|out|s=53|d=2606:4700:4700::1111 # Cloudflare IPv6 DNS Server # Include an external configuration file Include /etc/csf/csf.custom-config
rsyslog
- https://www.rsyslog.com/doc/reference/templates/templates-reserved-names.html#ref-templates-reserved-names
- https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/6/html/deployment_guide/s2-templates
Legacy
Send all logs to a rsyslog server and specify a port, @ is equal to using UDP. @@ is equal to TCP
# /etc/rsyslog.d/75-local-to-rsyslog-server.conf *.* @10.77.0.1:514
Custom template where hostname is defined, then sent to the syslog server - include the priority number as first extra variable
#/etc/rsyslog.d/70-local-to-rsyslog-server.conf $template SendHostname, "%PRI%1 %timestamp% myhost.mydomain.nl %syslogtag% %msg%\n" *.warning @10.77.0.1;SendHostname
Send messages to a syslog server, using a template aligned to IETF protocol 23
# /etc/rsyslog.d/61-qwe.conf *.* @10.77.0.1;RSYSLOG_SyslogProtocol23Format
Send messages to a syslog server, using a template aligned to IETF protocol 23, but specifying a custom hostname
# /etc/rsyslog.d/60-asd.conf $template custom_IETFprotocol_23,"%PRI%1 %TIMESTAMP:::date-rfc3339% prive.host.nl %APP-NAME% %PROCID% %MSGID% %STRUCTURED-DATA% %msg%\n" *.* @10.77.0.1;custom_IETFprotocol_23
Log to the local server with a static hostname, using a custom structure
# /etc/rsyslog.d/62-asd.conf $template NewHostname, "%timestamp% tester.mydomain.nl %syslogtag% %msg%\n" *.* /var/log/wewuzerrors.txt;NewHostname
An alternative to the contents above, specifying different/more fields
## /etc/rsyslog.d/65-customtemplate.conf # https://stackoverflow.com/questions/57890176/extending-rsyslogs-default-logging-template $template mynewtemplate,"%timegenerated% %HOSTNAME% %syslogfacility-text%.%syslogseverity-text% %syslogtag% %msg%\n" *.* /var/log/wazanda.txt;mynewtemplate
Rainerscript
Rainerscript: https://rsyslog.readthedocs.io/en/latest/rainerscript/control_structures.html
Write all local messages to a specific file
# /etc/rsyslog.d/60-asd.conf action(type="omfile" file="/var/log/isaidhey.txt")
Send message to a syslog server using IETF protocol 23
# /etc/rsyslog.d/70-local-to-rsyslog-server.conf
template(name="RSYSLOG_SyslogProtocol23Format" type="string"
string="<%PRI%>1 %TIMESTAMP:::date-rfc3339% %HOSTNAME% %APP-NAME% %PROCID% %MSGID% %STRUCTURED-DATA% %msg%\n")
# Send all logs to the target server
action(type="omfwd" Target="192.168.5.21" Template="RSYSLOG_SyslogProtocol23Format" Port="514" Protocol="udp")
Define a template aligned to IETF protocol 23 but specify a hostname to send as:
# /etc/rsyslog.d/71-local-to-rsyslog-server.conf
template(name="SendHostname" type="string"
string="<%PRI%>1 %TIMESTAMP:::date-rfc3339% myhost.mydomain.nl %APP-NAME% %PROCID% %MSGID% %STRUCTURED-DATA% %msg%\n")
# Send all logs to target syslog server and port
action(type="omfwd" Target="10.0.33.10" Template="SendHostname" Port="514" Protocol="udp")
Testing
# Use the logger tool to test syslog server reception logger -p local0.error 'Hello World!'
named
Checks
# Perform a test load of all primary zones within named.conf, as the named user sudo -u named named-checkconf -z # Check zone file 192.168.77.0 defined in the 77.168.192.in-addr.arpa zone named-checkzone 77.168.192.in-addr.arpa 192.168.77.0 # Check zone file brammerloo.nl defined in the brammerloo.nl zone named-checkzone brammerloo.nl brammerloo.nl
Configuration
Basic configuration for the options field in /etc/named.conf
options {
# Define on what IP to listen on, for port 53
listen-on port 53 { 127.0.0.1; 192.168.0.1; 192.168.1.1; };
directory "/var/named";
dump-file "/var/named/data/cache_dump.db";
statistics-file "/var/named/data/named_stats.txt";
memstatistics-file "/var/named/data/named_mem_stats.txt";
secroots-file "/var/named/data/named.secroots";
recursing-file "/var/named/data/named.recursing";
# Only allow DNS queries from specific local subnets
# To allow from anything use: allow query { any; };
allow-query { localhost; 127.0.0.1; 192.168.0.0/24; 192.168.1.0/24; };
# If the server can't resolve an address locally, use the following DNS servers for help
forwarders {
8.8.8.8;
1.1.1.1;
};
recursion yes;
dnssec-validation no;
managed-keys-directory "/var/named/dynamic";
geoip-directory "/usr/share/GeoIP";
pid-file "/run/named/named.pid";
session-keyfile "/run/named/session.key";
/* https://fedoraproject.org/wiki/Changes/CryptoPolicy */
include "/etc/crypto-policies/back-ends/bind.config";
};
Zone defnitions: /etc/named.rfc1912.zones
# Define zones to listen for
zone "brammerloo.nl" IN {
type master;
file "brammerloo.nl";
allow-update { none; };
};
zone "1.168.192.in-addr.arpa" IN {
type master;
file "192.168.1.0";
allow-update { none; };
};
Zone file for Reverse lookup: /var/named/192.168.1.0
$TTL 300
@ IN SOA ns1.brammerloo.nl. admin.brammerloo.nl. (
2023101102 ; serial
180 ; refresh
60 ; retry
108000 ; expire
60 ) ; minimum
IN NS ns1.brammerloo.nl.
; PTR Records
11 IN PTR node1.
21 IN PTR server1.
Zone file for domain: /var/named/brammerloo.nl
$TTL 300
@ IN SOA ns1.brammerloo.nl. admin.brammerloo.nl. (
2023101306 ; serial
180 ; refresh
60 ; retry
108000 ; expire
60 ) ; minimum
IN NS ns1.brammerloo.nl.
@ IN A 192.168.1.6 ; domain brammerloo.nl is me!
ns1.brammerloo.nl. IN A 192.168.78.31 ; FQDN for my domain
node1 IN A 192.168.78.31 ; Basic A-record
www IN CNAME node1 ; Point my website to my node1 A-record
dhcpd
dhclient
# Request DHCP addresses where applicable dhclient # Request an IPv4 adres from a DHCP server dhclient -4 # Show verbose information when requesting an IPv4 adres from a DHCP server dhclient -4 -v # Release a DHCP lease dhclient -r
Configuration
Basic configuration options in the /etc/dhcp/dhcpd.conf file
# Set the domain clients should use when resolving hostnames (equivalent to search domain)
option domain-name "brammerloo.nl";
# Set the domain name servers for DHCP clients
option domain-name-servers ns1.brammerloo.nl, 8.8.8.8;
default-lease-time 600;
max-lease-time 7200;
log-facility local7;
# Best practice = define any connected subnets, but don't configure DHCP for them
subnet 192.168.1.0 netmask 255.255.255.0 {
}
# Basic DHCP for a subnet configuration
subnet 192.168.0.0 netmask 255.255.255.0 {
range 192.168.0.100 192.168.0.150;
option routers 192.168.0.1;
}
smbd / Samba / CIFS
https://linuxconfig.org/install-samba-on-redhat-8
Checks
# List available shares on an IP or host smbclient -L //172.17.0.2 # Samba status checks smbstatus smbstatus -S smbstatus -b # Samba set debug mode smbcontrol smbd debug 1
Basic configuration
# Install and enable
dnf install samba samba-client
systemctl enable --now {smb,nmb}
# Create a client-user to authenticate with sudo useradd samba-user # Give the user a password to authenticate with sudo smbpasswd -a samba-user # Create a group to associate with the samba share sudo groupadd sambagroup # Add the user to the group we will be configuring for the share sudo usermod -a -G sambagroup samba-user # Create the folder we will be sharing sudo mkdir /var/shares/myshare # Apply proper permission sudo chown -R samba-user:sambagroup /var/shares/myshare/ sudo chmod -R 0770 /var/shares/myshare/ # Apply proper permission for SELinux sudo chcon -t samba_share_t /var/shares/myshare/ # Backup the default config cp /etc/samba/smb.conf /etc/samba/smb.conf~
# /etc/samba/smb.conf [global] workgroup = <DOMAIN-OR-WORKGROUP> server string = Samba Server %v netbios name = <SERVER-HOSTNAME> security = user map to guest = bad user dns proxy = no #==================== Share Definitions ====================== [share001] path = /var/shares/myshare valid users = @sambagroup guest ok = no writable = yes browsable = yes
# Reload Samba services
systemctl reload {smb,nmb}
# Mount in Windows
\\<SERVER-IP>\share001
user: samba-user pass: <Whatever password you filled in with smbpasswd -a
Docker
Checks
# List Docker containers docker ps # List all Docker container IDs docker ps -aq # List logs for container 987sdh3qrasdhj docker logs 987sdh3qrasdhj # List RAM/CPU usage for Docker container asdlkasd67k docker stats asdlkasd67k # Show verbose container information such as commands run, network, ID, etc docker inspect oiu2398sda87
Commands
# Enter the shell inside a docker container
docker exec -ti a89sd98sa7d /bin/bash
# Execute a command inside a container as a specific user, root in this case
docker exec -it -u root asd87289hasdadz tail /var/log/nginx/access.log
docker exec -u 0 -it as892asnj2as /bin/bash
# Restart docker container yoga
docker restart yoga
# Restart the 3 given containers
docker restart 79f71c7f4d91 bbb3d3f5c3b1 b0a3204d4098
# Start this container
docker start as9823nzxc0
# Stop this container
docker stop as9823nzxc0
# Restart all unhealthy Docker containers
for i in $(docker ps | grep unhealthy | awk '{print $1}'); do docker restart "$i"; done;
PowerDNS
- https://doc.powerdns.com/authoritative/index.html
- https://doc.powerdns.com/authoritative/manpages/pdns_server.1.html
- https://doc.powerdns.com/authoritative/manpages/pdnsutil.1.html
Checks
# List commands pdns_server --help # Check config and parse for errors pdns_server --config=check
# List available commands pdnsutil --help # Check config and parse for errors pdnsutil --config=check # List all available zones pdnsutil list-all-zones # List all domains in the primary zone pdnsutil list-all-zones primary # See zone information for a specific domain pdnsutil show-zone mydomain.com pdnsutil show-zone 77.5.10.in-addr.arpa # Check zone for errors pdnsutil check-zone mydomain.com # List all created TSIG keys pdnsutil list-tsig-keys
Commands
# Activate TSIG key for domain "myexample.com" in the primary zone pdnsutil " myexample.com transfer primary
MAAS
Checks
Logs in either place: /var/log/maas/ /var/snap/maas/common/log
# List status of MAAS services maas status # List MAAS commands maas --help # List available arguments for the init command maas init --help
Filesystems
# List clients connected to local mounts showmount
NFS
Checks
# NFS nfsstat # Detailed RPC and package information nfsstat -o all # Every RPC "program" is bound to a specific NFS version. Use NFS/CTDB logs in combination with the program ID to identify the failing component rpcinfo -p
Common
Exports
Use file /etc/exports to define exports to cliënts.
# Create the folders before exporting them mkdir -p /data/exports/customer1000/finance mkdir -p /data/exports/customer1001/backup
NFSv3 example:
#//////////////////////////////////////////////////////////////////////////////////////////// # Customer1000 /data/exports/customer1000/finance 192.168.20.1(rw,no_root_squash,sync) 192.168.20.2(rw,sync) #//////////////////////////////////////////////////////////////////////////////////////////// # Customer1001 /data/exports/customer1001/backup 192.168.30.1(rw,no_root_squash) 192.168.30.1(rw,no_root_squash,sync)
# Reload the NFS server to apply changes within /etc/exports systemctl reload nfs-server
Client mount
# Install NFS cliënt (Ubuntu) apt install nfs-common # Install NFS cliënt (RHEL) yum install nfs-utils # Mount NFS share located on server 192.168.20.1 on path /data/exports/customer1000/finance, to local server /mnt/nfs/ mount -v -t nfs 192.168.20.1:/data/exports/customer1000/finance /mnt/nfs/
Optimizations
Change these values depending on your usage and the available resources on your server.
# /etc/sysctl.d/nfs-tuning.conf net.core.rmem_max=1048576 net.core.rmem_default=1048576 net.core.wmem_max=1048576 net.core.wmem_default=1048576 net.ipv4.tcp_rmem=4096 1048576 134217728 net.ipv4.tcp_wmem=4096 1048576 134217728 vm.min_free_kbytes=8388608
# Reload above optimization sysctl -p /etc/sysctl.d/nfs-tuning.conf
Raise the number of NFS threads
# /etc/sysconfig/nfs # Number of nfs server processes to be started. # The default is 8. #RPCNFSDCOUNT=16 RPCNFSDCOUNT=128
Activate NFSD count on the fly
rpc.nfsd 64 # Check amount of threads /proc/fs/nfsd/threads
Ceph
Checks
# Display the running Ceph version ceph -v # Check the clusters' health and status ceph -s # Watch the clusters' health and status in real time ceph -w # Show detailed logs relating to cluster health ceph health detail # List configurations for a lot of stuff cephadm ls # List configurations for a lot of stuff ceph config dump # List all Ceph 'containers' and OSDs ceph orch ls # Lists all hosts, labels and basic host resource information ceph orch host ls --detail # List available storage devices ceph orch device ls # List all Ceph daemons ceph orch ps # List Ceph daemons of a specific type ceph orch ps --daemon_type=mgr # Show logs for a specific service ceph orch ls --service_name osd.all-available-devices --format yaml # Re-check the status of a host ceph cephadm check-host storage-3 # Check the current number of operations on a primary Ceph node ceph daemon /var/run/ceph/ceph-mds.xxxxxxxx.vxokby.asok dump_ops_in_flight
OSDs
# List all pools ceph osd lspools # See the status of all OSDs ceph osd stat # List all OSDs ceph osd tree # List all OSDs and related information in detail ceph osd df tree
PGs
# List all Placement Groups ceph pg dump # Check the status of Ceph PGs ceph pg stat
Authentication
# List all created clients and their permissions ceph auth ls # List permissions for a specific client ceph auth get client.cinder
Commands
# Enter the Ceph shell (single cluster) cephadm shell # Enter the Ceph shell for a specific cluster sudo /usr/sbin/cephadm shell --fsid asdjwqe-asjd324-asdki321-821asd-asd241-asdn1234- -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin2.keyring # Give node storage-4, which is already a cluster member, the admin tag ceph orch host label add storage-4 _admin
Installation (Quincy)
Using Cephadm: https://docs.ceph.com/en/quincy/cephadm/install/
Cephadm
# Create a folder for the cephadm tool mkdir cephadm cd cephadm/ # Download cephadm (Quincy) curl --silent --remote-name --location https://github.com/ceph/ceph/raw/quincy/src/cephadm/cephadm chmod +x cephadm # Output help ./cephadm -h # Install cephadm (Quincy) release ./cephadm add-repo --release quincy ./cephadm install # Check if cephadm is properly installed which cephadm
Bootstrap
# Bootstrap node and install Ceph cephadm bootstrap --mon-ip 192.168.100.11 # Check the status of the cluster cephadm shell -- ceph -s docker ps ## Optional # Enter the Ceph shell (single cluster) cephadm shell # Exit the Ceph shell exit # Install common Ceph packages/tools cephadm install ceph-common # Display the Ceph version ceph -v
Add additional hosts
# On your bootstrapped node create a key for SSH-access to the other hosts. ssh-keygen cat .ssh/id_rsa.pub # Add the newly generated key to the authorized_keys file for the relevant user, on the other hosts. # Copy the Ceph clusters' public key to the other nodes ssh-copy-id -f -i /etc/ceph/ceph.pub root@storage-2 ssh-copy-id -f -i /etc/ceph/ceph.pub root@storage-3 # Add the other nodes to the cluster, and assign them the admin role ceph orch host add storage-2 10.4.20.2 _admin ceph orch host add storage-3 10.4.20.3 _admin
Configuration
OSD creation
If you've installed ceph-osd on your host, this step will fail horribly with errors such as:
-1 bluestore(/var/lib/ceph/osd/ceph-1//block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1//block: (13) Permission denied -1 bdev(0x5571d5f69400 /var/lib/ceph/osd/ceph-1//block) open open got: (13) Permission denied -1 OSD::mkfs: ObjectStore::mkfs failed with error (13) Permission denied -1 ESC[0;31m ** ERROR: error creating empty object store in /var/lib/ceph/osd/ceph-0/: (13) Permission deniedESC[0m OSD, will rollback changes
# Configure all available storage to be used as OSD storage ceph orch apply osd --all-available-devices # Check for OSD problems watch ceph -s watch ceph osd tree
Delete pool
# Set ability to remove pools to true ceph config set mon mon_allow_pool_delete true # Remove the pool ceph osd pool rm tester tester --yes-i-really-really-mean-it # Set ability to remove pools to false ceph config set mon mon_allow_pool_delete false
Upgrade
Make sure your cluster status is healthy first!
# Upgrade Ceph to a specific version ceph orch upgrade start --ceph-version 17.2.0 # Check the status of the Ceph upgrade ceph orch upgrade status # Stop the Ceph upgrade ceph orch upgrade stop
Ceph client
Via Kernel
Mount a Ceph filesystem share using the kernel, Cephx and 3 mon hosts:
# Install common Ceph package for your distribution
apt-get install ceph-common
# Create and fill the ceph.conf file, mind the enter in the end
cat << 'EOF' >> /etc/ceph/ceph.conf
# minimal ceph.conf for 492f528f-90ae-49e0-b622-ae58b85e8cf0
[global]
fsid = 492f528f-90ae-49e0-b622-ae58b85e8cf0
mon_host = [v2:192.168.0.11:3300/0,v1:192.168.0.11:6789/0] [v2:192.168.0.12:3300/0,v1:192.168.0.12:6789/0] [v2:192.168.0.13:3300/0,v1:192.168.0.13:6789/0]
EOF
# Add the Cephx used by your user
cat << 'EOF' >> /etc/ceph/ceph.client.sofie.keyring
[client.sofie]
key = AIAOIWmaskjhqweASKhqwekjhASD==
EOF
# Mount your Ceph share by referring to the Ceph mons, the share on the Ceph mons, where you want to mount the share, and the userdata you use to connect to said share respectively.
mount -t ceph 192.168.0.11:6789,192.168:6789.0.12,192.168.0.13:6789:/shares/mycustomer/asd8asd8-as8d83-df4mjvjdf /mnt/ceph/mylocalsharelocation -o name=sofie
Ceph-fuse
# Mount a Ceph filesystem using the ceph-fuse client
apt install ceph-fuse
mkdir myshare/
nano sofie.keyring
[client.sofie]
key = AQCHc7tlvEUqOBasjdHASJD9Lma84nASDJqwe==
nano ceph.conf
[client]
client quota = true
mon host = 192.168.10.1:6789, 192.168.10.2:6789, 192.168.10.3:6789
sudo ceph-fuse ~/myshare \
--id=sofie \
--conf=./ceph.conf \
--keyring=./sofie.keyring \
--client-mountpoint=/volumes/_nogroup/6e99687f-asd2-47b0-8ba1-asduoiqwe/12398asnjd-0126-4cb3-9242-asduio1q23
# Debugmode in case of shit
sudo ceph-fuse ~/myshare \
--id=sofie \
--conf=./ceph.conf \
--keyring=./sofie.keyring \
--client-mountpoint=/volumes/_nogroup/6e99687f-asd2-47b0-8ba1-asduoiqwe/12398asnjd-0126-4cb3-9242-asduio1q23 -d -o debug
RBD-NBD
# List available volumes within the openstackvolumes pool rbd ls openstackhdd # List all available snapshots for object volume-asd9p12o3-90b2-1238-1209-as980d7213hs, which reside in pool openstackhdd rbd snap ls openstackhdd/volume-asd9p12o3-90b2-1238-1209-as980d7213hs # Map the volume-object to the local filesystem rbd-nbd map openstackhdd/volume-asd9p12o3-90b2-1238-1209-as980d7213hs # Map the volume-object as read-only to the local filesystem rbd-nbd map --read-only openstackhdd/volume-asd9p12o3-90b2-1238-1209-as980d7213hs # List currently mapped objects rbd-nbd list-mapped # Check what filesystem and partition the device contains fdisk -l /dev/nbd1 # Mount the device to a local folder mount /dev/nbd1p1 /mnt/storage # Unmount the device from the local folder umount /mnt/storage # 2 methods to unmap # Unmap the mapped object rbd-nbd unmap /dev/nbd2 # Unmap the mapped object rbd-nbd unmap volume-asd9p12o3-90b2-1238-1209-as980d7213hs
Remove node
# Remove running daemons ceph orch host drain storage-3 # Remove host from the cluster ceph orch host rm storage-3 # In storage-3, restart the node shutdown -r now
Destroy node
Scorched earth
Only execute if you want to annihilate your node and or cluster.
# Kill and destroy OSD 0 ceph osd down 0 && ceph osd destroy 0 --force # Stop Ceph services systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn@mon.host-1.service systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn@crash.host-1.service systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn@mgr.host-1.xmatqa.service systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn@mon.host-1.service systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn@node-exporter.host-1.service systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn@prometheus.host-1.service systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn.target # Disable Ceph services systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn@mon.host-1.service systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn@crash.host-1.service systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn@mgr.host-1.xmatqa.service systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn@mon.host-1.service systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn@node-exporter.host-1.service systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn@prometheus.host-1.service systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn.target # Destroy everything (packages, containers, configuration) ceph-deploy uninstall host-1 ceph-deploy purge host-1 rm -rf /var/lib/ceph # Check for failed services systemctl | grep ceph # Reset them so they disable properly systemctl reset-failed ceph-asd82asd-asd8-as92-a889-po89xc732cmn@prometheus.host-1.service # reboot shutdown -r now
BTRFS
Using LVM
# Install LVM creation tools depending on your OS yum install lvm2 apt install lvm2 # Check and note the disk you need fdisk -l # Format /dev/vdb as BTRFS echo -e "n\np\n1\n\n\nt\n8E\np\nw" | fdisk /dev/vdb # Create LVM pvcreate /dev/vdb1 vgcreate vdb_vg /dev/vdb1 lvcreate -l 100%FREE -n btrfs vdb_vg # Check pvs vgs # Create the BTRFS filesystem mkfs.btrfs /dev/vdb_vg/btrfs # Create a folder for the BTRFS mount mkdir -p /mnt/btrfs1 # Mount the BTRFS filesystem mount -t btrfs /dev/vdb_vg/btrfs /mnt/btrfs1/ # Modify fstab so the filesystem get mounted automatically on boot cat << 'EOF' >> /etc/fstab /dev/mapper/vdb_vg-btrfs /mnt/btrfs1 btrfs defaults 0 0 EOF
User management
# Create the books group groupadd books # Make myrthe part of the "philosophy" and "books" groups usermod myrthe -aG philosophy,books # See the groups myrthe is part of groups myrthe # The owner gains full control, group and everyone may: read, write and execute chmod 755 /home/ring/gollum.txt # Make ballrog the owner of the /data/sf4/cup folder chown ballrog:ballrog /data/sf4/cup # Make all files located anywhere within the .ssh, owned by the stalin user and soviet group chown -R stalin:soviet /home/stalin/.ssh # Delete the simba user and include his home folder and mail spool userdel -r simba
Create users and SSH access
Useradd
Variant #1 - RHEL
USER="danielle"
# Create user with a home-folder and add him to the wheel group
useradd -m -G wheel --shell /bin/bash ${USER}
# Create an SSH folder
mkdir -p /home/${USER}/.ssh
# Add a public key
cat << 'EOF' >> /home/${USER}/.ssh/authorized_keys
ssh-ed25519 123980idfas89132hadsckjh871234
EOF
# Set proper permissions for the .ssh folder and authorized_keys
chown -R ${USER}:${USER} /home/${USER}/.ssh
chmod 700 /home/${USER}/.ssh
chmod 600 /home/${USER}/.ssh/authorized_keys
Variant #2 - UBUNTU
USER="dylan"
# Create the admin group if it does not exist already
groupadd admin
# Create a group that is the same as the username
groupadd ${USER}
# Create user with a home-folder, set the primary group to his own group and add him to the admin group in addition, set default shell to /bin/bash
useradd -m -g ${USER} -G admin --shell /bin/bash ${USER}
# Create an SSH folder
mkdir -p /home/${USER}/.ssh
# Add the public key
cat << 'EOF' >> /home/${USER}/.ssh/authorized_keys
ssh-ed25519 AAAqhwekhakdhslsh8712398 keyname
EOF
# Set proper permissions for user and .ssh folder and authorized_keys
chown -R ${USER}:${USER} /home/${USER}/
chmod 700 /home/${USER}/.ssh
chmod 600 /home/${USER}/.ssh/authorized_keys
Adduser variant - Ubuntu
Create regular user and configure/add SSH public key:
USER="dylan"
# Create user and homefolder with a disabled password
adduser --disabled-password ${USER}
# Give the user sudo rights, do mind that you disabled his password in the previous command
usermod -aG sudo ${USER}
# Create an SSH folder
mkdir -p /home/${USER}/.ssh
# Add a public key
cat << 'EOF' >> /home/${USER}/.ssh/authorized_keys
ssh-ed25519 123980idfas89132hadsckjh871234
EOF
# Set proper permissions for user and .ssh folder and authorized_keys
chown -R ${USER}:${USER} /home/${USER}/
chmod 700 /home/${USER}/.ssh
chmod 600 /home/${USER}/.ssh/authorized_keys
Add Root user SSH keys
Add/configure SSH public key for root user, assuming no .ssh folder/file exists
# Add SSH-keys for root user echo "Creating authorized keys file and setting rights for root user" mkdir -p /root/.ssh # Add the following SSH keys to root cat << 'EOF' >> /root/.ssh/authorized_keys ssh-ed25519 AAAAC3NzaC1askdjasdsadsad mykeyname EOF chown -R root:root /root/.ssh chmod 700 /root/.ssh chmod 600 /root/.ssh/authorized_keys
Add regular user SSH keys
Add/configure SSH public key for a regular user, assuming no .ssh folder/file exists
USER="greed"
# Add SSH-keys to the defined user
echo "Creating authorized keys file and settings rights for ${USER} user"
mkdir -p /home/$USER/.ssh
# Add normal user SSH keys
cat << 'EOF' >> /home/$USER/.ssh/authorized_keys
ssh-ed25519 AAAAC3NzaC1askdjasdsadsad mykeyname
EOF
chown -R ${USER}:${USER} /home/$USER/.ssh
chmod 700 /home/$USER/.ssh
chmod 600 /home/$USER/.ssh/authorized_keys
Sudoers
Concerns /etc/sudoers
# Allow user jabami to execute any command, without specifying a passwd jabami ALL=(ALL) NOPASSWD: ALL # Allow user "chris" to perform the 2 given commands with sudo, no password. ## Define user and associate the command group variable "UPDATE_CMDS" chris ALL=(ALL) NOPASSWD: UPDATE_CMDS ## Define commands for the "UPDATE_CMDS" and "UPDATE_CMDS2" variables Cmnd_Alias UPDATE_CMDS = /usr/bin/apt-get update, /usr/bin/apt-get upgrade Cmnd_Alias UPDATE_CMDS2 = /usr/bin/apt-get update, /usr/bin/apt-get upgrade # Allow members of the group "researchers" to perform the commands in "UPDATE_CMDS2" with sudo rights, no password. ## User alias specification %researchers ALL=(ALL) NOPASSWD: UPDATE_CMDS2
Other
Performance tests
Network bandwidth & throughput
# Test bandwidth throughput with iperf # Listen on server-A on port 5101 iperf3 -s -p 5101 # Connect to server-A from server-B iperf3 -c 192.168.0.1 -p 5101
Filesystem
# Testing disk/share throughput # Create "testfile" of size 1710x1M in current folder time dd if=/dev/zero of=testfile bs=1M count=1710 # Create "'testfile2" of size 5x1G in current folder time dd if=/dev/zero of=testfile2 bs=1G count=5 # Show copy-time of "testfile" to disk or share time cp testfile /mnt/btfrs/data/<LOCATION>/ # Methods of testing disk or share throughput # show read-time from the mount to null time cat /mnt/btfrs/data/<FILE> > /dev/null # show copy-time from the mount to null time dd if=/mnt/btfrs/data/<FILE> of=/dev/null bs=1M # show copy-time from the mount to the current folder time cp /mnt/btfrs/data/<FILE> . # Copy one folder to another with rsync while showing progress rsync -avhW --no-compress --progress <source>/ <destination>/
Create different temp folder
# Create a temporary TMP folder mkdir -p /scratch/tmp/ # Activate temporary TMP folder export TMPDIR=/scratch/tmp
