Skip to main content

Installing NixOS (Encrypted BTRFS Opt-in)

Intro

This details the steps to create a LUKS encrypted BTRFS NixOS installation with impermanance, only keeping the settings that are required in between reboots.

Features:

  • UEFI Boot w/GRUB
  • Encrypted Btrfs File System with hourly snapshots on /home and /var/local
  • Swap File instead of Swap Partition w/Hibernate Support

This is a work in progress using the standard configuration.nix setup.

Todo
  • Setup FDE instead of Partition 2 Errors
  • Determine a need for Impermanance module

configuration.nix

Setup Variables

DISK_EFI_SIZE_MB=512
DISK=/dev/vda
PASSWORD=password

Wipe and Partition Disk (Swap File)

wipefs "${DISK}" -a -f
sgdisk --zap-all "${DISK}"
sgdisk --clear \
                --new=1:0:+"${DISK_EFI_SIZE_MB}"MiB --typecode=1:ef00 --change-name=1:EFI \
                --new=2:0:0 --typecode=2:8200 --change-name=2:pool0_0 \
                "${DISK}"
EFI_PARTITION=1
CRYPT_PARTITION=2

Create Encrypted Disk and Initial Filesystems

echo "${PASSWORD}" | cryptsetup --verify-passphrase -v luksFormat "${CRYPT_PARTITION}"
echo "${PASSWORD}" | cryptsetup open "${DISK}""${CRYPT_PARTITION}" pool0_0
mkfs.btrfs -f /dev/mapper/pool0_0
mkfs.vfat "${DISK}""${EFI_PARTITION}"

Create BTRFS Subvolumes

mount -t btrfs /dev/mapper/pool0_0 /mnt
btrfs subvolume create /mnt/root
mkdir -p /mnt/home
btrfs subvolume create /mnt/home/active
btrfs subvolume create /mnt/home/snapshots
btrfs subvolume create /mnt/nix
btrfs subvolume create /mnt/persist
btrfs subvolume create /mnt/swap
mkdir -p /mnt/var_local
btrfs subvolume create /mnt/var_local/active
btrfs subvolume create /mnt/var_local/snapshots
btrfs subvolume create /mnt/var_log

Take a readonly snapshot of the root subvolume, which will get rolled back to on every boot.

btrfs subvolume snapshot -r /mnt/root /mnt/root-blank

Mount Subvolumes and Partitions

umount /mnt
mount -o subvol=root,compress=zstd,noatime /dev/mapper/pool0_0 /mnt

mkdir -p /mnt/home
mount -o subvol=home/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home
mkdir -p /mnt/home/.snapshots
mount -o subvol=home/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home/.snapshots

mkdir -p /mnt/nix
mount -o subvol=nix,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/nix

mkdir -p /mnt/persist
mount -o subvol=persist,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/persist

mkdir -p /mnt/swap
mount -o subvol=swap,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/swap

mkdir -p /mnt/var/local
mount -o subvol=var_local/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local
mkdir -p /mnt/var/local/.snapshots
mount -o subvol=var_local/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local/.snapshots

mkdir -p /mnt/var/log
mount -o subvol=var_log,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/log

mkdir -p /mnt/boot/efi
mount -o defaults,nosuid,nodev,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro /dev/disk/by-partlabel/EFI /mnt/boot/efi

Generate NixOS Hardware Configuration

nixos-generate-config --root /mnt

File Based Swap

sed -i '1h;1!H;$!d;x;s/.*fileSystems.*};\n[^\n]*/&   \
  swapDevices = [{\
    device = \"\/swap\/swapfile\";\
    size = (1024 * 16) + (1024 * 2); # RAM size + 2 GB\n  }];\n/' /mnt/etc/nixos/hardware-configuration.nix

sed -i '/swapDevices = \[ \];/d' /mnt/etc/nixos/hardware-configuration.nix

truncate -s 0 /mnt/swap/swapfile
chattr +C /mnt/swap/swapfile

sed -i "/boot.extraModulePackages/a\\
  boot.kernelParams = [ \"resume_offset=$(btrfs inspect-internal map-swapfile -r /mnt/swap/swapfile)\" ];\n\
  boot.resumeDevice = \"/dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2)\" ;"  /mnt/etc/nixos/hardware-configuration.nix

Force var_log subvolume to be available for boot

sed -i "/subvol=var_log/a\      neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix

Force BTRFS mount options

Nixos-generate config isn't smart enough to add options as per here.

sed -i "s|options = \[ \(.*\) \];|options = \[ \1 \"compress=zstd\" \"noatime\"  \];|g" /mnt/etc/nixos/hardware-configuration.nix 
sed -i 's|"subvol=swap" "compress=zstd"|"subvol=swap" "compress=none"|g' /mnt/etc/nixos/hardware-configuration.nix

Allow discards to encrypted root

sed -i '/boot.initrd.luks.devices/a\  boot.initrd.luks.devices."pool0_0".allowDiscards = true;' /mnt/etc/nixos/hardware-configuration.nix

Create Base Configuration

/etc/nixos/configuration.nix

{ config, pkgs, ... }:

{
  imports =
    [ 
      ./hardware-configuration.nix
    ];

  boot = {
    loader = {
      efi = {
        canTouchEfiVariables = false;
      };
      grub = {
        enable = true;
        device = "nodev";  # No device for EFI
        efiSupport = true;
        enableCryptodisk = false;
        useOSProber = false;
        efiInstallAsRemovable = true; # in case canTouchEfiVariables doesn't work for your system
      };
    };

    supportedFilesystems = [ 
      "btrfs"
      "fat" "vfat" "exfat" "ntfs" # Microsoft 
      "cifs"                      # Windows Network Share
    ];
  };

  networking = {
    hostName = "beef"; 
    networkmanager.enable = true;
  };

  i18n.defaultLocale = "en_US.UTF-8"; 

  security.sudo.extraConfig = ''
    Defaults lecture = never
  '';
  
  services = {
    cinnamon.apps.enable = false ;
    btrbk = {
      instances."btrbak" = {
        onCalendar = "*-*-* *:00:00";
        settings = {
          timestamp_format = "long";
          snapshot_preserve_min = "2d";
          preserve_day_of_week = "sunday" ;
          preserve_hour_of_day = "0" ;
          target_preserve = "48h 10d 4w 12m 10y" ;
          volume."/home" = {
            snapshot_create = "always";
            subvolume = ".";
            snapshot_dir = ".snapshots";
          };
          volume."/var/local" = {
            snapshot_create = "always";
            subvolume = ".";
            snapshot_dir = ".snapshots";
          };
    };
    
    openssh = {
      enable = true;
      settings = {
        PermitRootLogin = "yes" ;
      };
    };
    xserver = { 
      enable = true;
      layout = "us";
      libinput.enable = true;

      displayManager.gdm.enable = true ;
      desktopManager.cinnamon.enable = true;
    };
  };

  system.stateVersion = "23.05";

  systemd.services = {
    create-swapfile = {
      serviceConfig.Type = "oneshot";
      wantedBy = [ "swap-swapfile.swap" ];
      script = ''
        swapfile="/swap/swapfile"
        if [ -f "$swapfile" ]; then
            echo "Swap file $swapfile already exists, taking no action"
        else
            echo "Setting up swap file $swapfile"
            ${pkgs.coreutils}/bin/truncate -s 0 "$swapfile"
            ${pkgs.e2fsprogs}/bin/chattr +C "$swapfile"
        fi
      '';
    };
  };
  

  time.timeZone = "America/Vancouver";

  users.users.dave = {
     description = "Dave Conroy";
     createHome = true ;
     home = "/home/dave" ;
     shell = "/bin/sh" ; # This is actually bash
     group = "users" ;
     extraGroups = [ "wheel" "docker" ];
     uid = 2323;
     isNormalUser = true;
     hashedPassword = "$y$j9T$jEYLXjGrR06/tp76fxyDq/$mX4GTWL7CjVXgAcS5nAHEiT6WIH8uD/IfXj16fuTRQ1";
     packages = with pkgs; [
     ];
   };

}

Install NixOS

nixos-install

Flakes

Booting into new system

Configure system and make any changes. Afterwords, run script below to detect changes in snapshot vs root since boot and start working on Persistence mapping.

Create differences script

cat <<EOF > /home/$USER/nix-finddiff

#!/usr/bin/env bash

_tmp_root=$(mktemp -d)
mkdir -p "${_tmp_root}"
mount -o subvol=/ /dev/mapper/pool0_0 "${_tmp_root}" > /dev/null 2>&1

set -euo pipefail

OLD_TRANSID=$(sudo btrfs subvolume find-new ${_tmp_root}/root-blank 9999999)
OLD_TRANSID=${OLD_TRANSID#transid marker was }

sudo btrfs subvolume find-new "${_tmp_root}/root" "$OLD_TRANSID" | sed '$d' | cut -f17- -d' ' | sort | uniq | 
while read path; do
    path="/$path"
    if [ -L "$path" ]; then
        : # The path is a symbolic link, so is probably handled by NixOS already
    elif [ -d "$path" ]; then
        : # The path is a directory, ignore
    else
        echo "$path"
    fi
done
umount "${_tmp_root}"
rm -rf "${_tmp_root}"

EOF

chmod +x /home/$USER/nix-finddiff

Persistence Rules

Bluetooth

  systemd.tmpfiles.rules = [
    "L /var/lib/bluetooth - - - - /persist/var/lib/bluetooth"
  ];

##
sudo mkdir -p /persist/var/lib/bluetooth

Docker

  virtualisation = {
     docker.enable = true;
  };
  
    systemd.tmpfiles.rules = [
    "L /var/lib/docker - - - - /persist/var/lib/docker"
  ];

##
sudo systemctl stop docker
sudo mkdir -p /persist/var/lib/
sudo cp -r {,/persist}/var/lib/docker

Network Manager

  environment.etc."NetworkManager/system-connections" = {
    source = "/persist/etc/NetworkManager/system-connections/";
  };

  systemd.tmpfiles.rules = [
    "L /var/lib/NetworkManager/secret_key - - - - /persist/var/lib/NetworkManager/secret_key"
    "L /var/lib/NetworkManager/seen-bssids - - - - /persist/var/lib/NetworkManager/seen-bssids"
    "L /var/lib/NetworkManager/timestamps - - - - /persist/var/lib/NetworkManager/timestamps"
  ];
  
##
sudo mkdir -p /persist/etc/NetworkManager
sudo cp -r {,/persist}/etc/NetworkManager/system-connections
sudo mkdir -p /persist/var/lib/NetworkManager
sudo cp /var/lib/NetworkManager/{secret_key,seen-bssids,timestamps} /persist/var/lib/NetworkManager/

NixOS

 environment.etc = {
    nixos.source = "/persist/etc/nixos";
    NIXOS.source = "/persist/etc/NIXOS";
    machine-id.source = "/persist/etc/machine-id";
  };
    
sudo cp -R {,/persist}/etc/nixos
sudo cp -R {,/persist}/etc/NIXOS
sudo cp {,/persist}/etc/machine-id

OpenSSH

  services.openssh = {
    enable = true;
    hostKeys = [
      {
        path = "/persist/etc/ssh/ssh_host_ed25519_key";
        type = "ed25519";
      }
      {
        path = "/persist/etc/ssh/ssh_host_rsa_key";
        type = "rsa";
        bits = 4096;
      }
    ];
  };

##
sudo mkdir -p /persist/etc/ssh
sudo cp -R /etc/ssh/ssh_host* /persist/etc/ssh/

Scratchpad

Unmount commands

umount /mnt/boot
umount /mnt/home
umount /mnt/nix
umount /mnt/persist
umount /mnt/var/local
umount /mnt/var/log
umount /mnt

Restart Install over and over

umount /mnt/boot
umount /mnt/home
umount /mnt/nix
umount /mnt/persist
umount /mnt/swap
umount /mnt/var/local
umount /mnt/var/log
umount /mnt

PASSWORD=password
DISK_EFI_SIZE_MB=512
DISK=/dev/vda
wipefs "${DISK}" -a -f
sgdisk --zap-all "${DISK}"
sgdisk --clear \
                --new=1:0:+"${DISK_EFI_SIZE_MB}"MiB --typecode=1:ef00 --change-name=1:EFI \
                --new=2:0:0 --typecode=2:8300 --change-name=2:pool0_0 \
                "${DISK}"
EFI_PARTITION=1
CRYPT_PARTITION=2
echo "${PASSWORD}" | cryptsetup luksFormat "${DISK}""${CRYPT_PARTITION}"
echo "${PASSWORD}" | cryptsetup open "${DISK}""${CRYPT_PARTITION}" pool0_0
mkfs.btrfs -f /dev/mapper/pool0_0
mkfs.vfat "${DISK}""${EFI_PARTITION}"
mount -t btrfs /dev/mapper/pool0_0 /mnt
btrfs subvolume create /mnt/root
mkdir -p /mnt/home
btrfs subvolume create /mnt/home/active
btrfs subvolume create /mnt/home/snapshots
btrfs subvolume create /mnt/nix
btrfs subvolume create /mnt/persist
btrfs subvolume create /mnt/swap
mkdir -p /mnt/var_local
btrfs subvolume create /mnt/var_local/active
btrfs subvolume create /mnt/var_local/snapshots
btrfs subvolume create /mnt/var_log
btrfs subvolume snapshot -r /mnt/root /mnt/root-blank
umount /mnt
mount -o subvol=root,compress=zstd,noatime /dev/mapper/pool0_0 /mnt
mkdir -p /mnt/home
mount -o subvol=home/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home
mkdir -p /mnt/home/.snapshots
mount -o subvol=home/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home/.snapshots
mkdir -p /mnt/nix
mount -o subvol=nix,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/nix
mkdir -p /mnt/persist
mount -o subvol=persist,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/persist
mkdir -p /mnt/swap
mount -o subvol=swap,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/swap
mkdir -p /mnt/var/local
mount -o subvol=var_local/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local
mkdir -p /mnt/var/local/.snapshots
mount -o subvol=var_local/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local/.snapshots
mkdir -p /mnt/boot
mount -o defaults,nosuid,nodev,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro /dev/disk/by-partlabel/EFI /mnt/boot
nixos-generate-config --root /mnt

sed -i '1h;1!H;$!d;x;s/.*fileSystems.*};\n[^\n]*/&   \
  swapDevices = [{\
    device = \"\/swap\/swapfile\";\
    size = (1024 * 16) + (1024 * 2); # RAM size + 2 GB\n  }];\n/' /mnt/etc/nixos/hardware-configuration.nix

sed -i '/swapDevices = \[ \];/d' /mnt/etc/nixos/hardware-configuration.nix

truncate -s 0 /mnt/swap/swapfile
chattr +C /mnt/swap/swapfile

sed -i "/boot.extraModulePackages/a\\
  boot.kernelParams = [ \"resume_offset=$(btrfs inspect-internal map-swapfile -r /mnt/swap/swapfile)\" ];\n\
  boot.resumeDevice = \"/dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2)\" ;"  /mnt/etc/nixos/hardware-configuration.nix

sed -i "/subvol=var_log/a\      neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix 
sed -i "s|options = \[ \(.*\) \];|options = \[ \1 \"compress=zstd\" \"noatime\"  \];|g" /mnt/etc/nixos/hardware-configuration.nix 
sed -i 's|"subvol=swap" "compress=zstd"|"subvol=swap" "compress=none"|g' /mnt/etc/nixos/hardware-configuration.nix 
sed -i '/boot.initrd.luks.devices/a\  boot.initrd.luks.devices."pool0_0".allowDiscards = true;' /mnt/etc/nixos/hardware-configuration.nix 
rm -rf /mnt/etc/nixos/configuration.nix
nano /mnt/etc/nixos/configuration.nix



   nixos-rebuild boot
   sudo mkdir -p /persist/var/lib/
   sudo cp -r {,/persist}/var/lib/docker
   nano /etc/nixos/configuration.nix 
   sudo mkdir -p /persist/etc/NetworkManager
   sudo cp -r {,/persist}/etc/NetworkManager/system-connections
   sudo mkdir -p /persist/var/lib/NetworkManager
   sudo cp /var/lib/NetworkManager/{secret_key,seen-bssids,timestamps} /persist/var/lib/NetworkManager/  
   sudo mkdir -p /persist/etc/ssh
   sudo cp -R /etc/ssh/ssh_host* /persist/etc/ssh/
   sudo cp -R {,/persist}/etc/nixos
   sudo cp -R {,/persist}/etc/NIXOS
   sudo cp {,/persist}/etc/machine-id
   reboot

Errors

Encrypted /boot

When mounting the ESP partition at /boot/efi instead of /boot grub picks the mounted pool0_0 filesystem and puts it in its configuration. Idea - Creating a seperate LUKS boot partition??

Back to top