systemd Reference: Unit Files, journalctl, Timers, Socket Activation & Boot Analysis

# Status and inspection:
systemctl status nginx               # status + recent journal output
systemctl status nginx.service       # same — .service is implied
systemctl is-active nginx            # active / inactive / failed (for scripts)
systemctl is-enabled nginx           # enabled / disabled / static / masked
systemctl list-units --state=failed  # all failed units

# Start / stop / restart:
systemctl start   nginx
systemctl stop    nginx
systemctl restart nginx              # stop + start (brief downtime)
systemctl reload  nginx              # reload config without restart (if supported)
systemctl reload-or-restart nginx    # reload if running, restart if stopped

# Enable / disable (on boot):
systemctl enable nginx               # enable on boot (creates symlink)
systemctl enable --now nginx         # enable + start immediately (atomic)
systemctl disable nginx
systemctl disable --now nginx        # disable + stop immediately

systemctl mask   nginx               # prevent start (even manually)
systemctl unmask nginx

# Daemon reload (after editing unit files):
systemctl daemon-reload              # ALWAYS run after editing /etc/systemd/system/*.service

# /etc/systemd/system/myapp.service
[Unit]
Description=My Application
After=network.target postgresql.service    # start AFTER these
Requires=postgresql.service               # if postgresql fails, myapp fails too
# Wants= is softer than Requires= (doesn't fail if dependency missing)

[Service]
Type=simple           # process stays in foreground (most common for modern apps)
# Type=forking        # process daemonizes (forks to background) — use for old-style daemons
# Type=notify         # process signals systemd when ready (sd_notify)
# Type=oneshot        # runs once and exits (for scripts/tasks)

User=myapp
Group=myapp
WorkingDirectory=/opt/myapp
ExecStart=/usr/bin/node /opt/myapp/server.js
ExecReload=/bin/kill -HUP $MAINPID    # what "reload" runs

# Environment:
Environment="NODE_ENV=production" "PORT=3000"
EnvironmentFile=/etc/myapp/environment   # key=value file (one per line)
# NEVER put secrets in Environment= — visible in systemctl show + /proc

# Restart policy:
Restart=always                           # restart on any exit
Restart=on-failure                       # restart only on non-zero exit or signal
RestartSec=5                             # wait 5 seconds before restart

# Output:
StandardOutput=journal                   # send stdout to journald
StandardError=journal
SyslogIdentifier=myapp                   # how it appears in journalctl -t myapp

# Security hardening:
NoNewPrivileges=yes
PrivateTmp=yes                           # isolated /tmp
ReadOnlyPaths=/
ReadWritePaths=/var/lib/myapp /var/log/myapp

[Install]
WantedBy=multi-user.target              # starts in normal multi-user mode

# systemctl edit creates a drop-in at /etc/systemd/system/nginx.service.d/override.conf
# Doesn't modify the package-provided unit file (survives package upgrades!)
systemctl edit nginx                   # opens $EDITOR

# Example drop-in (increase memory limit, add env var):
[Service]
MemoryMax=2G
Environment="ADDITIONAL_ENV=value"
Restart=always
RestartSec=3

# View final merged config:
systemctl cat nginx                    # shows original + all drop-ins
systemctl show nginx                   # shows all properties with defaults

# To REPLACE (not just override) a section, first clear it:
[Service]
ExecStart=                             # empty line clears the existing value
ExecStart=/usr/sbin/nginx -c /etc/nginx/nginx-custom.conf

# /etc/systemd/system/backup.service
[Unit]
Description=Database Backup

[Service]
Type=oneshot
ExecStart=/usr/local/bin/backup.sh
User=backup

# /etc/systemd/system/backup.timer
[Unit]
Description=Daily backup at 3am

[Timer]
OnCalendar=*-*-* 03:00:00             # daily at 3am
# OnCalendar=Mon *-*-* 09:00:00       # every Monday at 9am
# OnCalendar=*-*-01 00:00:00          # first day of each month
Persistent=true                        # run immediately on startup if last run was missed
RandomizedDelaySec=300                 # add random 0-5 min delay (avoids herd effect)
Unit=backup.service                    # which service to activate

[Install]
WantedBy=timers.target

# Enable + verify:
systemctl enable --now backup.timer
systemctl list-timers                  # see all timers, next trigger time
systemctl status backup.timer          # status + last triggered
journalctl -u backup.service           # logs from the service run by the timer

# OnCalendar syntax:
# yearly, quarterly, monthly, weekly, daily, hourly
# Mon..Fri *-*-* 09:00
# *-*-* 00/6:00:00  → every 6 hours
systemd-analyze calendar "*-*-* 03:00:00"   # verify calendar expression

# Follow a service's logs:
journalctl -u nginx -f                 # follow
journalctl -u nginx --since "1 hour ago"
journalctl -u nginx --since "2026-03-14 10:00" --until "2026-03-14 11:00"
journalctl -u nginx -n 100             # last 100 lines
journalctl -u nginx --no-pager         # for piping to grep

# Filter by priority:
journalctl -u nginx -p err             # errors and above
journalctl -p 0..3                     # emerg(0) alert(1) crit(2) err(3)
journalctl -p warning                  # warning and above

# Boot logs:
journalctl -b                          # current boot
journalctl -b -1                       # previous boot
journalctl --list-boots                # all boots with timestamps and IDs

# System-wide filtering:
journalctl -k                          # kernel messages (like dmesg)
journalctl -t myapp                    # by syslog identifier
journalctl _UID=1000                   # by user ID
journalctl _PID=12345                  # by PID

# JSON output (for log aggregation):
journalctl -u nginx -o json | jq '.MESSAGE'
journalctl -u nginx -o json-pretty | head -50

# Disk usage + maintenance:
journalctl --disk-usage
journalctl --vacuum-size=500M          # keep only 500MB of logs
journalctl --vacuum-time=30d           # delete logs older than 30 days

# Targets (like runlevels):
systemctl get-default                  # current default target
systemctl set-default multi-user.target   # change default
systemctl isolate rescue.target        # switch to single-user (immediate)

# Key targets:
# multi-user.target  = normal server mode (no GUI)
# graphical.target   = multi-user + GUI
# rescue.target      = single user, minimal services
# emergency.target   = minimal (even less than rescue)
# network.target     = network interfaces are up
# network-online.target = network has connectivity (important!)

# Boot time analysis:
systemd-analyze                        # total boot time
systemd-analyze blame                  # which units took longest
systemd-analyze critical-chain         # critical path of boot sequence
systemd-analyze plot > boot.svg        # visual timeline (open in browser)

# Dependency graph:
systemd-analyze dot nginx.service | dot -Tsvg > nginx-deps.svg
systemctl list-dependencies nginx      # tree view of dependencies
systemctl list-dependencies --reverse nginx  # what depends on nginx

# Socket activation (service only starts when a connection arrives):
# nginx.socket activates nginx.service on connection to port 80
[Socket]
ListenStream=80              # TCP port
Accept=no                    # pass socket to service (not per-connection fork)

[Install]
WantedBy=sockets.target

# Path unit (watch a file/directory):
[Path]
PathModified=/etc/myapp/config.yaml   # trigger on file modification
Unit=myapp-reload.service

# Mount unit (replaces /etc/fstab entries):
[Mount]
What=/dev/sdb1
Where=/mnt/data
Type=ext4
Options=defaults,noatime

WantedBy=multi-user.target
# Filename MUST match mount point: mnt-data.mount (replace / with -)

# Scope and slice — resource control:
# Limit memory for a slice of services:
# /etc/systemd/system/myapp.slice
[Slice]
MemoryMax=4G
CPUQuota=200%                # 200% = 2 full CPU cores