Linux: Difference between revisions
From Cheatsheet
Jump to navigationJump to search
(→Checks) |
No edit summary |
||
| Line 7: | Line 7: | ||
* [[Linux:Network]] | * [[Linux:Network]] | ||
* [[Linux:Tools]] | * [[Linux:Tools]] | ||
* [[Linux:Filesystems]] | |||
== Common checks == | == Common checks == | ||
Revision as of 11:38, 30 October 2023
This page is or will soon be an amalgamation of content from other pages
Common checks
Monitoring
# See CPU + RAM usage, system stats and open processes top # Only list processes making active use of the CPU top -i # Only list processes making active use of the CPU, and include the entire command being instead of just the tool-name top -ci # Prettier version of top that can be customized htop # Reimagined version of top, includes network and disk usage by default btop # List all running processes ps aux
Systemd
# Open journalctl at the beginning journalctl -b # Open journalctl at the end journalctl -e # Open journalctl but include service information journalctl -x # Show journalctl logs for the sshd service, starting from the end journalctl -u sshd -e # Output contents directly to the shell journalctl --no-pager
OS & Distribution
# Print OS and host information hostnamectl # Show OS and distribution information cat /proc/version # Show OS and distribution information cat /etc/os-release # Print distribution-specific information lsb_release -a
Hardware & kernel
# List installed kernel modules lsmod # Print Kernel messages dmesg # Print Kernel messages with humanized timestamps dmesg -T # SCSI hardware information cat /proc/scsi/scsi # Print hardware/BIOS information dmidecode # Print hardware/BIOS information of a specific type dmidecode -t 1 # List all connected hardware lshw # List physical network hardware lshw -short -class network # List physical memory hardware lshw -class memory # Show PCI information lspci # Show verbose PCI information lspci -v # List all block/filesystem devices lsblk # List block devices and partition tables fdisk -l
Pacemaker
# Show status of the pacemaker cluster pcs cluster status # Show status of the pacemaker service pcs status # Show configured pacemaker resources pcs resource config # Show a specific configured resource pcs resource show ResourceNameHere
Services
Common
systemctl
# List all services that are running or exited
systemctl
# List all services, running or otherwise
systemctl --all
# List all failed services
systemctl --state=failed
# Reset the failed service "nginx"
systemctl reset-failed nginx
# View the status of the "nfs-server" service
systemctl status nfs-server
# Output the config file of "rsyslog" to the shell
systemctl cat rsyslog
# Restart the "sshd" service, terminating established connections and re-parsing the configuration
systemctl restart sshd
# Reload the "nginx" service so that it only re-parses the configuration
systemctl reload nginx
# Stop the "nfs-ganesha" service so that it stops being run
systemctl stop nfs-ganesha
# Start the "nfs-ganesha" service so that it starts being run again
systemctl start nfs-ganesha
# Disable the "mariadb" service so that it doesn't start after the next boot
systemctl disable mariadb
# Enable the "mariadb" service so that it starts after the next boot.
systemctl enable mariadb
# Check the logs for all failed services
for i in $(systemctl --state=failed | head -n -4 | tail -n +2 | awk '{print $1}'); do systemctl --no-pager status "$i"; done
NTP
Timedatectl
# Show the current status of timedatectl timedatectl # List available timezones timedatectl list-timezones # Set the timezone to Amsterdam timedatectl set-timezone Europe/Amsterdam # Show verbose sync information timedatectl timesync-status
SNMP
V3 client installation
apt install snmpd snmp libsnmp-dev cp /etc/snmp/snmpd.conf /etc/snmp/snmpd.conf.bak systemctl stop snmpd net-snmp-create-v3-user -ro -X <CRYPTO-PASSWORD> -a SHA -X <PASSWORD> -x AES <USERNAME>
# /etc/snmp/snmpd.conf sysLocation NL;Zuid-Holland;Rotterdam, 78 MyStreet;2nd Floor;Server Room;Rack sysContact Me <me@example.org> agentaddress 192.168.0.10
systemctl start snmpd systemctl enable snmpd
# Test snmpwalk -v3 -a SHA -A "AUTHENTICATION PASSWORD" -x AES -X "CRYPTO PASSWORD" -l authPriv -u "MYUSER" localhost | head
CTDB
Checks
# Verify CTDB cluster status ctdb status # Show the allocated IP addresses and to which nodes they're bound ctdb ip # See the status of all CTDB-scripts ctdb scriptstatus ctdb event status # Show the time of the last failover the duration it took to recover ctdb uptime # See various statistics and data ctdb statistics # Use the onnode command to execute a command on all cluster nodes onnode all ctdb status
Commands
# Stop a ctdb cluster member ctdb stop # Start a stopped ctdb cluster member ctdb continue
Firewalls
UFW
Checks
# Show summary of UFW status ufw status # Show verbose UFW status ufw status verbose # Show UFW rules numbered ufw status numbered
Commands
# Allow access from a specific IP to a port and add a comment that show in the status ufw allow from 10.0.0.253 to any port 22 proto tcp comment 'Allow SSH access from XYZ location' # Delete numbered Firewall rule 56 ufw delete 56 # Disable UFW logging (prevent syslog spam) ufw logging off # Set UFW logging back to the default ufw logging low
Firewalld
SNMP access
# /etc/firewalld/services/snmp.xml <?xml version="1.0" encoding="utf-8"?> <service> <short>SNMP</short> <description>SNMP protocol</description> <port protocol="udp" port="161"/> </service>
firewall-cmd --reload firewall-cmd --zone=public --add-service snmp --permanent firewall-cmd --reload
Firewall-cmd
Checks
# List all available commands firewall-cmd -h # Check the configuration file of the firewall for errors firewall-cmd --check-config # Display the current state of firewall-cmd (running/shutdown) firewall-cmd --state # Display all available zones firewall-cmd --get-zones # List all whitelisted services firewall-cmd --list-services # List all services you can potentially enable firewall-cmd --get-services # List all added or enabled services and ports in more detail firewall-cmd --list-all # List verbose information for all zones firewall-cmd --list-all-zones # List verbose information for the public zone firewall-cmd --list-all --zone=public # See what port(s) are associated with the dns service firewall-cmd --info-service dns # List all opened ports firewall-cmd --list-ports # List kernel ruleset generated for nftables(?) nft list ruleset
Commands
# Reload the firewall firewall-cmd --reload # Whitelist the dns service, persistently even after reboot firewall-cmd --add-service=dns ; sudo firewall-cmd --runtime-to-permanent; firewall-cmd --reload # Whitelist the http service, persistently even after reboot firewall-cmd --add-service=http ; sudo firewall-cmd --runtime-to-permanent; firewall-cmd --reload # Remove the http service from the whitelist firewall-cmd --remove-service=http # Add port 1234 (tcp) to the whitelist firewall-cmd --add-port=1234/tcp # Remove port 1234 (tcp) from the whitelist firewall-cmd --remove-port=1234/tcp # Add port 2345 (udp) to the whitelist in zone external firewall-cmd --zone=external --add-port=2345/udp # Remove port 2345 (udp) from the whitelist for zone external firewall-cmd --zone=external --add-port=2345/udp # Add current configuration to configuration permanently firewall-cmd –runtime-to-permanent
DANGEROUS
# SHUT IT DOWN DOC - DROP ALL PACKETS AND EXPIRE EXISTING CONNECTIONS firewall-cmd --panic-on # ACCEPT PACKETS AGAIN firewall-cmd --panic-off
CSF
ConfigServer Security and Firewall
General
- Common configuration: /etc/csf/csf.conf
- Blacklist: /etc/csf/csf.deny
- Whitelist: /etc/csf/csf.allow
Installation
From the official instructions: https://download.configserver.com/csf/install.txt
Prerequisites
Perl Modules ============ While most should be installed on a standard perl installation the following may need to be installed manually: # On rpm based systems: yum install perl-libwww-perl.noarch perl-LWP-Protocol-https.noarch perl-GDGraph # On APT based systems: apt-get install libwww-perl liblwp-protocol-https-perl libgd-graph-perl
Install
cd /usr/src rm -fv csf.tgz wget https://download.configserver.com/csf.tgz tar -xzf csf.tgz cd csf sh install.sh # Next, test whether you have the required iptables modules: perl /usr/local/csf/bin/csftest.pl # Don't worry if you cannot run all the features, so long as the script doesn't report any FATAL errors
Checks
# Check the running status of csf csf status
Commands
# Commit config changes by restarting csf csf -r
csf.conf
Some common changes within the configuration file
# Set testing to 0 when your CSF configuration is 'production' ready TESTING = "0" # Allow access to any service you're hosting locally, for example https TCP_IN = "443" UDP_IN = "" # Allow all outwards HTTP/HTTPS traffic so you can yum/apt update TCP_OUT = "80,443" # Allow outgoing traceroute UDP_OUT = "33434:33523" # Allow your server to be pinged ICMP_IN = "0"
Formatting
The varying styles of formatting used in allow.conf
# Allow anything relating to the following IPs/ranges 192.168.10.0/24 # Our application breaks without this range 192.168.1.1 # Our gateway or something # Detailed entries based on Transport protocol, direction, Application protocol and IP tcp:in:d=22:s=7.7.7.7 # SSH access from our VPN udp:in:d=161:s=10.11.12.100 # SNMP Access tcp|in|d=22|s=fe80::1:/16 # IPV6 SSH access from our jumpgateway udp|in|d=3389|s=10.1.0.0/24 # RDP Access from our entire office range tcp|out|d=80,443|d=1.2.3.4/32 # Allow outgoing HTTP/HTTPS access via port 80 and 443 # Allow sending Syslog messages to our Syslog server udp|out|d=514|d=192.168.20.5 # UDP syslog server tcp|out|d=10514|d=192.168.20.5 # UDP syslog server # Allow sending queries to some DNS servers tcp|out|s=53|d=8.8.8.8 udp|out|s=53|d=1.1.1.1 udp|out|s=53|d=2606:4700:4700::1111 # Cloudflare IPv6 DNS Server # Include an external configuration file Include /etc/csf/csf.custom-config
rsyslog
- https://www.rsyslog.com/doc/reference/templates/templates-reserved-names.html#ref-templates-reserved-names
- https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/6/html/deployment_guide/s2-templates
Legacy
Send all logs to a rsyslog server and specify a port, @ is equal to using UDP. @@ is equal to TCP
# /etc/rsyslog.d/75-local-to-rsyslog-server.conf *.* @10.77.0.1:514
Custom template where hostname is defined, then sent to the syslog server - include the priority number as first extra variable
#/etc/rsyslog.d/70-local-to-rsyslog-server.conf $template SendHostname, "%PRI%1 %timestamp% myhost.mydomain.nl %syslogtag% %msg%\n" *.warning @10.77.0.1;SendHostname
Send messages to a syslog server, using a template aligned to IETF protocol 23
# /etc/rsyslog.d/61-qwe.conf *.* @10.77.0.1;RSYSLOG_SyslogProtocol23Format
Send messages to a syslog server, using a template aligned to IETF protocol 23, but specifying a custom hostname
# /etc/rsyslog.d/60-asd.conf $template custom_IETFprotocol_23,"%PRI%1 %TIMESTAMP:::date-rfc3339% prive.host.nl %APP-NAME% %PROCID% %MSGID% %STRUCTURED-DATA% %msg%\n" *.* @10.77.0.1;custom_IETFprotocol_23
Log to the local server with a static hostname, using a custom structure
# /etc/rsyslog.d/62-asd.conf $template NewHostname, "%timestamp% tester.mydomain.nl %syslogtag% %msg%\n" *.* /var/log/wewuzerrors.txt;NewHostname
An alternative to the contents above, specifying different/more fields
## /etc/rsyslog.d/65-customtemplate.conf # https://stackoverflow.com/questions/57890176/extending-rsyslogs-default-logging-template $template mynewtemplate,"%timegenerated% %HOSTNAME% %syslogfacility-text%.%syslogseverity-text% %syslogtag% %msg%\n" *.* /var/log/wazanda.txt;mynewtemplate
Rainerscript
Rainerscript: https://rsyslog.readthedocs.io/en/latest/rainerscript/control_structures.html
Write all local messages to a specific file
# /etc/rsyslog.d/60-asd.conf action(type="omfile" file="/var/log/isaidhey.txt")
Send message to a syslog server using IETF protocol 23
# /etc/rsyslog.d/70-local-to-rsyslog-server.conf
template(name="RSYSLOG_SyslogProtocol23Format" type="string"
string="<%PRI%>1 %TIMESTAMP:::date-rfc3339% %HOSTNAME% %APP-NAME% %PROCID% %MSGID% %STRUCTURED-DATA% %msg%\n")
# Send all logs to the target server
action(type="omfwd" Target="192.168.5.21" Template="RSYSLOG_SyslogProtocol23Format" Port="514" Protocol="udp")
Define a template aligned to IETF protocol 23 but specify a hostname to send as:
# /etc/rsyslog.d/71-local-to-rsyslog-server.conf
template(name="SendHostname" type="string"
string="<%PRI%>1 %TIMESTAMP:::date-rfc3339% myhost.mydomain.nl %APP-NAME% %PROCID% %MSGID% %STRUCTURED-DATA% %msg%\n")
# Send all logs to target syslog server and port
action(type="omfwd" Target="10.0.33.10" Template="SendHostname" Port="514" Protocol="udp")
Testing
# Use the logger tool to test syslog server reception logger -p local0.error 'Hello World!'
named
Checks
# Perform a test load of all primary zones within named.conf, as the named user sudo -u named named-checkconf -z # Check zone file 192.168.77.0 defined in the 77.168.192.in-addr.arpa zone named-checkzone 77.168.192.in-addr.arpa 192.168.77.0 # Check zone file brammerloo.nl defined in the brammerloo.nl zone named-checkzone brammerloo.nl brammerloo.nl
Configuration
Basic configuration for the options field in /etc/named.conf
options {
# Define on what IP to listen on, for port 53
listen-on port 53 { 127.0.0.1; 192.168.0.1; 192.168.1.1; };
directory "/var/named";
dump-file "/var/named/data/cache_dump.db";
statistics-file "/var/named/data/named_stats.txt";
memstatistics-file "/var/named/data/named_mem_stats.txt";
secroots-file "/var/named/data/named.secroots";
recursing-file "/var/named/data/named.recursing";
# Only allow DNS queries from specific local subnets
# To allow from anything use: allow query { any; };
allow-query { localhost; 127.0.0.1; 192.168.0.0/24; 192.168.1.0/24; };
# If the server can't resolve an address locally, use the following DNS servers for help
forwarders {
8.8.8.8;
1.1.1.1;
};
recursion yes;
dnssec-validation no;
managed-keys-directory "/var/named/dynamic";
geoip-directory "/usr/share/GeoIP";
pid-file "/run/named/named.pid";
session-keyfile "/run/named/session.key";
/* https://fedoraproject.org/wiki/Changes/CryptoPolicy */
include "/etc/crypto-policies/back-ends/bind.config";
};
Zone defnitions: /etc/named.rfc1912.zones
# Define zones to listen for
zone "brammerloo.nl" IN {
type master;
file "brammerloo.nl";
allow-update { none; };
};
zone "1.168.192.in-addr.arpa" IN {
type master;
file "192.168.1.0";
allow-update { none; };
};
Zone file for Reverse lookup: /var/named/192.168.1.0
$TTL 300
@ IN SOA ns1.brammerloo.nl. admin.brammerloo.nl. (
2023101102 ; serial
180 ; refresh
60 ; retry
108000 ; expire
60 ) ; minimum
IN NS ns1.brammerloo.nl.
; PTR Records
11 IN PTR node1.
21 IN PTR server1.
Zone file for domain: /var/named/brammerloo.nl
$TTL 300
@ IN SOA ns1.brammerloo.nl. admin.brammerloo.nl. (
2023101306 ; serial
180 ; refresh
60 ; retry
108000 ; expire
60 ) ; minimum
IN NS ns1.brammerloo.nl.
@ IN A 192.168.1.6 ; domain brammerloo.nl is me!
ns1.brammerloo.nl. IN A 192.168.78.31 ; FQDN for my domain
node1 IN A 192.168.78.31 ; Basic A-record
www IN CNAME node1 ; Point my website to my node1 A-record
dhcpd
dhclient
# Request DHCP addresses where applicable dhclient # Request an IPv4 adres from a DHCP server dhclient -4 # Show verbose information when requesting an IPv4 adres from a DHCP server dhclient -4 -v # Release a DHCP lease dhclient -r
Configuration
Basic configuration options in the /etc/dhcp/dhcpd.conf file
# Set the domain clients should use when resolving hostnames (equivalent to search domain)
option domain-name "brammerloo.nl";
# Set the domain name servers for DHCP clients
option domain-name-servers ns1.brammerloo.nl, 8.8.8.8;
default-lease-time 600;
max-lease-time 7200;
log-facility local7;
# Best practice = define any connected subnets, but don't configure DHCP for them
subnet 192.168.1.0 netmask 255.255.255.0 {
}
# Basic DHCP for a subnet configuration
subnet 192.168.0.0 netmask 255.255.255.0 {
range 192.168.0.100 192.168.0.150;
option routers 192.168.0.1;
}
smbd / Samba / CIFS
https://linuxconfig.org/install-samba-on-redhat-8
Checks
# List available shares on an IP or host smbclient -L //172.17.0.2 # Samba status checks smbstatus smbstatus -S smbstatus -b # Samba set debug mode smbcontrol smbd debug 1
Basic configuration
# Install and enable
dnf install samba samba-client
systemctl enable --now {smb,nmb}
# Create a client-user to authenticate with sudo useradd samba-user # Give the user a password to authenticate with sudo smbpasswd -a samba-user # Create a group to associate with the samba share sudo groupadd sambagroup # Add the user to the group we will be configuring for the share sudo usermod -a -G sambagroup samba-user # Create the folder we will be sharing sudo mkdir /var/shares/myshare # Apply proper permission sudo chown -R samba-user:sambagroup /var/shares/myshare/ sudo chmod -R 0770 /var/shares/myshare/ # Apply proper permission for SELinux sudo chcon -t samba_share_t /var/shares/myshare/ # Backup the default config cp /etc/samba/smb.conf /etc/samba/smb.conf~
# /etc/samba/smb.conf [global] workgroup = <DOMAIN-OR-WORKGROUP> server string = Samba Server %v netbios name = <SERVER-HOSTNAME> security = user map to guest = bad user dns proxy = no #==================== Share Definitions ====================== [share001] path = /var/shares/myshare valid users = @sambagroup guest ok = no writable = yes browsable = yes
# Reload Samba services
systemctl reload {smb,nmb}
# Mount in Windows
\\<SERVER-IP>\share001
user: samba-user pass: <Whatever password you filled in with smbpasswd -a
Docker
Checks
# List Docker containers docker ps # List all Docker container IDs docker ps -aq # List logs for container 987sdh3qrasdhj docker logs 987sdh3qrasdhj # List RAM/CPU usage for Docker container asdlkasd67k docker stats asdlkasd67k # Show verbose container information such as commands run, network, ID, etc docker inspect oiu2398sda87
Commands
# Enter the shell inside a docker container
docker exec -ti a89sd98sa7d /bin/bash
# Execute a command inside a container as a specific user, root in this case
docker exec -it -u root asd87289hasdadz tail /var/log/nginx/access.log
docker exec -u 0 -it as892asnj2as /bin/bash
# Restart docker container yoga
docker restart yoga
# Restart the 3 given containers
docker restart 79f71c7f4d91 bbb3d3f5c3b1 b0a3204d4098
# Start this container
docker start as9823nzxc0
# Stop this container
docker stop as9823nzxc0
# Restart all unhealthy Docker containers
for i in $(docker ps | grep unhealthy | awk '{print $1}'); do docker restart "$i"; done;
PowerDNS
- https://doc.powerdns.com/authoritative/index.html
- https://doc.powerdns.com/authoritative/manpages/pdns_server.1.html
- https://doc.powerdns.com/authoritative/manpages/pdnsutil.1.html
Checks
# List commands pdns_server --help # Check config and parse for errors pdns_server --config=check
# List available commands pdnsutil --help # Check config and parse for errors pdnsutil --config=check # List all available zones pdnsutil list-all-zones # List all domains in the primary zone pdnsutil list-all-zones primary # See zone information for a specific domain pdnsutil show-zone mydomain.com pdnsutil show-zone 77.5.10.in-addr.arpa # Check zone for errors pdnsutil check-zone mydomain.com # List all created TSIG keys pdnsutil list-tsig-keys
Commands
# Activate TSIG key for domain "myexample.com" in the primary zone pdnsutil " myexample.com transfer primary
MAAS
Checks
Logs in either place: /var/log/maas/ /var/snap/maas/common/log
# List status of MAAS services maas status # List MAAS commands maas --help # List available arguments for the init command maas init --help
Filesystems
# List clients connected to the local filesystem showmount
SMB/CIFS checks
# Samba checks smbstatus smbstatus -S smbstatus -b # Samba set debug mode smbcontrol smbd debug 1
NFS
Checks
# NFS nfsstat # Detailed RPC and package information nfsstat -o all # Every RPC "program" is bound to a specific NFS version. Use NFS/CTDB logs in combination with the program ID to identify the failing component rpcinfo -p
Common
Exports
Use file /etc/exports to define exports to cliënts.
# Create the folders before exporting them mkdir -p /data/exports/customer1000/finance mkdir -p /data/exports/customer1001/backup
NFSv3 example:
#//////////////////////////////////////////////////////////////////////////////////////////// # Customer1000 /data/exports/customer1000/finance 192.168.20.1(rw,no_root_squash,sync) 192.168.20.2(rw,sync) #//////////////////////////////////////////////////////////////////////////////////////////// # Customer1001 /data/exports/customer1001/backup 192.168.30.1(rw,no_root_squash) 192.168.30.1(rw,no_root_squash,sync)
# Reload the NFS server to apply changes within /etc/exports systemctl reload nfs-server
Mount
# Install NFS cliënt (Ubuntu) apt install nfs-common # Install NFS cliënt (RHEL) yum install nfs-utils # Mount NFS share located on server 192.168.20.1 on path /data/exports/customer1000/finance, to local server /mnt/nfs/ mount -v -t nfs 192.168.20.1:/data/exports/customer1000/finance /mnt/nfs/
Optimizations
Change these values depending on your usage and the available resources on your server.
# /etc/sysctl.d/nfs-tuning.conf net.core.rmem_max=1048576 net.core.rmem_default=1048576 net.core.wmem_max=1048576 net.core.wmem_default=1048576 net.ipv4.tcp_rmem=4096 1048576 134217728 net.ipv4.tcp_wmem=4096 1048576 134217728 vm.min_free_kbytes=8388608
# Reload above optimization sysctl -p /etc/sysctl.d/nfs-tuning.conf
Raise the number of NFS threads
# /etc/sysconfig/nfs # Number of nfs server processes to be started. # The default is 8. #RPCNFSDCOUNT=16 RPCNFSDCOUNT=128
Activate NFSD count on the fly
rpc.nfsd 64 # Check amount of threads /proc/fs/nfsd/threads
Ceph
Checks
# Display the running Ceph version ceph -v # Check the clusters' health and status ceph -s # Watch the clusters' health and status in real time ceph -w # Show detailed logs relating to cluster health ceph health detail # List all Ceph 'containers' and OSDs ceph orch ls # List available storage devices ceph orch device ls # Show logs for a specific service ceph orch ls --service_name osd.all-available-devices --format yaml # Re-check the status of a host ceph cephadm check-host storage-3
# List all pools ceph osd lspools # See the status of all OSDs ceph osd stat # List all OSDs ceph osd tree # List all OSDs and related information in detail ceph osd df tree
# List all Placement Groups ceph pg dump # Check the status of Ceph PGs ceph pg stat
Commands
# Enter the Ceph shell (single cluster) cephadm shell
Installation
Using Cephadm: https://docs.ceph.com/en/quincy/cephadm/install/
Cephadm
# Create a folder for the cephadm tool mkdir cephadm cd cephadm/ # Download cephadm (Quincy) curl --silent --remote-name --location https://github.com/ceph/ceph/raw/quincy/src/cephadm/cephadm chmod +x cephadm # Output help ./cephadm -h # Install cephadm (Quincy) release ./cephadm add-repo --release quincy ./cephadm install # Check if cephadm is properly installed which cephadm
Bootstrap
# Bootstrap node and install Ceph cephadm bootstrap --mon-ip 192.168.100.11 # Check the status of the cluster cephadm shell -- ceph -s docker ps ## Optional # Enter the Ceph shell (single cluster) cephadm shell # Exit the Ceph shell exit # Install common Ceph packages/tools cephadm install ceph-common # Display the Ceph version ceph -v
Add additional hosts
# On your bootstrapped node create a key for SSH-access to the other hosts. ssh-keygen cat .ssh/id_rsa.pub # Add the newly generated key to the authorized_keys file for the relevant user, on the other hosts. # Copy the Ceph clusters' public key to the other nodes ssh-copy-id -f -i /etc/ceph/ceph.pub root@storage-2 ssh-copy-id -f -i /etc/ceph/ceph.pub root@storage-3 # Add the admin role to the other nodes ceph orch host add storage-2 10.4.20.2 _admin ceph orch host add storage-3 10.4.20.3 _admin
OSD creation
If you've installed ceph-osd on your host, this step will fail horribly with errors such as:
-1 bluestore(/var/lib/ceph/osd/ceph-1//block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1//block: (13) Permission denied -1 bdev(0x5571d5f69400 /var/lib/ceph/osd/ceph-1//block) open open got: (13) Permission denied -1 OSD::mkfs: ObjectStore::mkfs failed with error (13) Permission denied -1 ESC[0;31m ** ERROR: error creating empty object store in /var/lib/ceph/osd/ceph-0/: (13) Permission deniedESC[0m OSD, will rollback changes
# Configure all available storage to be used as OSD storage ceph orch apply osd --all-available-devices # Check for OSD problems watch ceph -s watch ceph osd tree
Commands
# Enter the Ceph shell for a specific cluster sudo /usr/sbin/cephadm shell --fsid asdjwqe-asjd324-asdki321-821asd-asd241-asdn1234- -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin2.keyring # Give node storage-4, which is already a cluster member, the admin tag ceph orch host label add storage-4 _admin # Mount a Ceph filesystem with 3 mon hosts, using a secretfile # Contents in the secretfile is ONLY the secret / key mount -t ceph 192.168.0.11,192.168.0.12,192.168.0.13:/shares/mycustomer/asd8asd8-as8d83-df4mjvjdf /mnt/ceph-storage -o name=customershare-28,secretfile=/etc/ceph/customer28-secretfile
Upgrade
Make sure your cluster status is healthy first!
# Upgrade Ceph to a specific version ceph orch upgrade start --ceph-version 17.2.0 # Check the status of the Ceph upgrade ceph orch upgrade status # Stop the Ceph upgrade ceph orch upgrade stop
RBD-NBD
# List available volumes within the openstackvolumes pool rbd ls openstackhdd # List all available snapshots for object volume-asd9p12o3-90b2-1238-1209-as980d7213hs, which resides in pool ghgvolumes rbd snap ls openstackhdd/volume-asd9p12o3-90b2-1238-1209-as980d7213hs # Map the volume-object to the local filesystem rbd-nbd map openstackhdd/volume-asd9p12o3-90b2-1238-1209-as980d7213hs # Map the volume-object as read-only to the local filesystem rbd-nbd map --read-only openstackhdd/volume-asd9p12o3-90b2-1238-1209-as980d7213hs # List currently mapped objects rbd-nbd list-mapped # Check what filesystem and partition the device contains fdisk -l /dev/nbd1 # Mount the device to a local folder mount /dev/nbd1p1 /mnt/storage # Unmount the device from the local folder umount /mnt/storage # 2 methods to unmap # Unmap the mapped object rbd-nbd unmap /dev/nbd2 # Unmap the mapped object rbd-nbd unmap volume-asd9p12o3-90b2-1238-1209-as980d7213hs
Remove node
# Remove running daemons ceph orch host drain storage-3 # Remove host from the cluster ceph orch host rm storage-3 # In storage-3, restart the node and restart shutdown -r now
Destroy node
Scorched earth
Only execute if you want to annihalate your node and or cluster.
# Kill and destroy OSD 0 ceph osd down 0 && ceph osd destroy 0 --force # Stop Ceph services systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn@mon.host-1.service systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn@crash.host-1.service systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn@mgr.host-1.xmatqa.service systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn@mon.host-1.service systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn@node-exporter.host-1.service systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn@prometheus.host-1.service systemctl stop ceph-asd82asd-asd8-as92-a889-po89xc732cmn.target # Disable Ceph services systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn@mon.host-1.service systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn@crash.host-1.service systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn@mgr.host-1.xmatqa.service systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn@mon.host-1.service systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn@node-exporter.host-1.service systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn@prometheus.host-1.service systemctl disable ceph-asd82asd-asd8-as92-a889-po89xc732cmn.target # Destroy everything (packages, containers, configuration) ceph-deploy uninstall host-1 ceph-deploy purge host-1 rm -rf /var/lib/ceph # Check for failed services systemctl | grep ceph # Reset them so they disable properly systemctl reset-failed ceph-asd82asd-asd8-as92-a889-po89xc732cmn@prometheus.host-1.service # reboot shutdown -r now
BTRFS
Using LVM
# Install LVM creation tools depending on your OS yum install lvm2 apt install lvm2 # Check and note the disk you need fdisk -l # Format /dev/vdb as BTRFS echo -e "n\np\n1\n\n\nt\n8E\np\nw" | fdisk /dev/vdb # Create LVM pvcreate /dev/vdb1 vgcreate vdb_vg /dev/vdb1 lvcreate -l 100%FREE -n btrfs vdb_vg # Check pvs vgs # Create the BTRFS filesystem mkfs.btrfs /dev/vdb_vg/btrfs # Create a folder for the BTRFS mount mkdir -p /mnt/btrfs1 # Mount the BTRFS filesystem mount -t btrfs /dev/vdb_vg/btrfs /mnt/btrfs1/ # Modify fstab so the filesystem get mounted automatically on boot cat << 'EOF' >> /etc/fstab /dev/mapper/vdb_vg-btrfs /mnt/btrfs1 btrfs defaults 0 0 EOF
User management
# Create the books group groupadd books # Make myrthe part of the "philosophy" and "books" groups usermod myrthe -aG philosophy,books # See the groups myrthe is part of groups myrthe # The owner gains full control, group and everyone may: read, write and execute chmod 755 /home/ring/gollum.txt # Make ballrog the owner of the /data/sf4/cup folder chown ballrog:ballrog /data/sf4/cup # Make all files located anywhere within the .ssh, owned by the stalin user and soviet group chown -R stalin:soviet /home/stalin/.ssh # Delete the simba user and include his home folder and mail spool userdel -r simba
Create user (RHEL)
# Create user with a home-folder and add him to the wheel group useradd -m john -g wheel # Set a password for the john user passwd john # Create the SSH folder for john mkdir -p /home/john/.ssh # Add a public key to john's account echo "ssh rsa-123980idfas89132hadsckjh871234" >> /home/john/.ssh/authorized_keys # Set proper permissions for the .ssh folder and authorized_keys chown -R john:john /home/john/.ssh chmod 700 /home/john/.ssh chmod 600 /home/john/.ssh/authorized_keys
Sudoers
Concerns /etc/sudoers
# Allow user jabami to execute any command, without specifying a passwd jabami ALL=(ALL) NOPASSWD: ALL # Allow user "chris" to perform the 2 given commands with sudo, no password. ## Define user and associate the command group variable "UPDATE_CMDS" drake ALL=(ALL) NOPASSWD: UPDATE_CMDS ## Define commands for the "UPDATE_CMDS" variable Cmnd_Alias UPDATE_CMDS = /usr/bin/apt-get update, /usr/bin/apt-get upgrade # Allow members of the group "researchers" to perform the 2 given commands with sudo, no password. ## User alias specification %researchers ALL=(ALL) NOPASSWD: UPDATE_CMDS2 ## Define commands for the "UPDATE_CMDS2" variable Cmnd_Alias UPDATE_CMDS2 = /usr/bin/apt-get update, /usr/bin/apt-get upgrade
Other
Throughput test
# Test bandwidth throughput with iperf # Listen on server-A on port 5101 iperf3 -s -p 5101 # Connect to server-A from server-B iperf3 -c 192.168.0.1 -p 5101
# Testing disk/share throughput # Create "testfile" of size 1710x1M in current folder time dd if=/dev/zero of=testfile bs=1M count=1710 # Create "'testfile2" of size 5x1G in current folder time dd if=/dev/zero of=testfile2 bs=1G count=5 # Show copy-time of "testfile" to disk or share time cp testfile /mnt/btfrs/data/<LOCATION>/ # Methods of testing disk or share throughput # show read-time from the mount to null time cat /mnt/btfrs/data/<FILE> > /dev/null # show copy-time from the mount to null time dd if=/mnt/btfrs/data/<FILE> of=/dev/null bs=1M # show copy-time from the mount to the current folder time cp /mnt/btfrs/data/<FILE> . # Copy one folder to another with rsync while showing progress rsync -avhW --no-compress --progress <source>/ <destination>/
Create different temp folder
# Create a temporary TMP folder mkdir -p /scratch/tmp/ # Activate temporary TMP folder export TMPDIR=/scratch/tmp
