Linux — Thinking Like a System Engineer

# Browse the filesystem hierarchy
ls /
ls /etc | head -20
ls /var/log

# Check disk usage per directory
du -sh /* 2>/dev/null | sort -h

# See inode number
ls -i file.txt

# Check inode usage on a filesystem
df -i

# Create a hard link
ln original.txt hardlink.txt

# Create a symbolic link
ln -s /etc/nginx/nginx.conf nginx.conf

# Find all broken symlinks
find /etc -type l ! -exec test -e {} \; -print

# Change permissions
chmod 755 script.sh        # rwxr-xr-x
chmod u+x,g-w script.sh   # symbolic mode

# Change ownership
chown alice:devs file.txt
chown -R alice:devs /var/www/

# View effective permissions
stat file.txt
namei -l /path/to/file    # trace permissions along path

# Check current umask
umask         # e.g. 0022

# New file permissions = 0666 - 0022 = 0644 (rw-r--r--)
# New dir permissions  = 0777 - 0022 = 0755 (rwxr-xr-x)

# Set umask in /etc/profile or ~/.bashrc
umask 027     # more restrictive — group can read, others nothing

# Edit sudoers safely (validates syntax before saving)
visudo

# Allow user to run specific commands without password
alice ALL=(ALL) NOPASSWD: /usr/bin/systemctl restart nginx

# Check what you're allowed to run
sudo -l

# Run as a different user (not root)
sudo -u www-data id

# View process tree
pstree -p
ps auxf

# Find process by name
pgrep -a nginx
pidof nginx

# Send signals
kill -15 1234   # SIGTERM — graceful shutdown
kill -9 1234    # SIGKILL — force kill (no cleanup)
kill -1 1234    # SIGHUP — reload config (for many daemons)
kill -0 1234    # check if process exists (no signal sent)

# List all running services
systemctl list-units --type=service --state=running

# Start / stop / restart / reload
systemctl start nginx
systemctl stop nginx
systemctl restart nginx
systemctl reload nginx    # graceful config reload

# Enable at boot / disable
systemctl enable nginx
systemctl disable nginx

# Check status with logs
systemctl status nginx -l

# Show dependencies
systemctl list-dependencies nginx

# After editing unit files
systemctl daemon-reload
systemctl restart myapp

# Check why a service failed
systemctl status myapp
journalctl -u myapp -n 50 --no-pager

# Check restart count
systemctl show myapp --property=NRestarts

# Reset failure counter
systemctl reset-failed myapp

# All logs, newest first
journalctl -r

# Follow live (like tail -f)
journalctl -f

# Since last boot
journalctl -b

# Specific service
journalctl -u nginx -n 100

# Time range
journalctl --since "2024-01-01 10:00" --until "2024-01-01 11:00"

# Priority levels (err and above)
journalctl -p err -b

# Show kernel messages only
journalctl -k

# Disk usage of journal
journalctl --disk-usage

# Vacuum old logs
journalctl --vacuum-size=500M

# Show all mounted filesystems
df -h
lsblk
mount | column -t

# Check /etc/fstab (filesystems to mount at boot)
cat /etc/fstab

# Mount manually
mount /dev/sdb1 /mnt/data

# Mount with options
mount -o ro,noexec /dev/sdc1 /mnt/backup

# Check disk space
df -h

# Check inode usage — can be full even when space is free!
df -i

# Find what's eating space
du -sh /var/* 2>/dev/null | sort -rh | head -20
ncdu /var           # interactive (install with apt/yum)

# Find files larger than 100MB
find / -type f -size +100M 2>/dev/null

# Find directories with most files (inode issue)
find /tmp -type d -exec sh -c 'echo "$(ls -A "$1" | wc -l) $1"' _ {} \; | sort -rn | head

# Memory overview
free -h
vmstat 1 5      # 5 samples, 1 second apart

# Detailed memory stats
cat /proc/meminfo

# Top memory consumers
ps aux --sort=-%mem | head -10

# CPU load vs CPU usage
uptime              # load average: 1, 5, 15 min
top                 # interactive
htop                # better interactive

# Per-CPU stats
mpstat -P ALL 1     # (from sysstat package)

# Generate key pair
ssh-keygen -t ed25519 -C "your-email@example.com"

# Copy public key to server
ssh-copy-id -i ~/.ssh/id_ed25519.pub user@server

# Manual copy (if ssh-copy-id unavailable)
cat ~/.ssh/id_ed25519.pub | ssh user@server "mkdir -p ~/.ssh && cat >> ~/.ssh/authorized_keys"

# Connect with specific key
ssh -i ~/.ssh/id_ed25519 user@server

# Connect with verbose output (debugging)
ssh -v user@server

# Local forwarding — access remote service locally
# Access server's port 5432 (postgres) as localhost:5432
ssh -L 5432:localhost:5432 user@server

# Remote forwarding — expose local port on remote server
# Expose local :8080 as server's :9090
ssh -R 9090:localhost:8080 user@server

# Dynamic forwarding (SOCKS proxy)
ssh -D 1080 user@server

# Jump host / bastion
ssh -J bastion-user@bastion.example.com target-user@private-server

systemctl status myapp
journalctl -u myapp --since "5 minutes ago"
# Run ExecStart command manually
# Check: ports in use, missing files, wrong user
ss -tlnp | grep :8080

# Who owns the file?
ls -la /path/to/file

# What's the process running as?
ps aux | grep myapp

# Trace permission along path
namei -l /path/to/file

# Check SELinux/AppArmor if permissions look fine
getenforce          # SELinux
aa-status           # AppArmor
ausearch -m avc -ts recent  # SELinux audit log

df -h       # which filesystem is full?
df -i       # check inodes too

# Find large files
du -sh /var/* | sort -rh | head
find /var/log -name "*.log" -size +100M

# Quick fixes
journalctl --vacuum-size=100M
truncate -s 0 /var/log/huge.log  # zero out (don't delete if open)

uptime          # load average
top             # interactive — press 1 for per-CPU
ps aux --sort=-%cpu | head -10

# Is it CPU-bound or I/O-bound?
vmstat 1 5
# procs r column = processes waiting for CPU
# io wa column = I/O wait percentage
# High wa + low cpu = I/O bottleneck
# High cpu = CPU bottleneck

iostat -x 1 5   # per-disk I/O stats