Skip to main content

Installing NixOS (Encrypted BTRFS Impermanance)

Intro

This details the steps to create a LUKS encrypted BTRFS NixOS installation with impermanance, only keeping the settings that are required in between reboots.

Features:

  • UEFI Boot w/GRUB
  • Encrypted Btrfs File System with hourly snapshots on /home and /var/local
  • Swap File instead of Swap Partition w/Hibernate Support
  • Works on Single Disk and RAID setups

Disk Setup

Setup Variables

DISK_EFI_SIZE_MB=512
DISK_SWAP_SIZE_GB=4
DISK=/dev/vda
PASSWORD=password
If setting up RAID1:
DISK2=/dev/vdb

Parition Disk

Choose one, Swap parititon needs to be used if using any sort of RAID

Wipe and Partition Disk (Swap Partition)

wipefs "${DISK}" -a -f
sgdisk --zap-all "${DISK}"
sgdisk --clear \
                --new=1:0:+"${DISK_EFI_SIZE_MB}"MiB --typecode=1:ef00 --change-name=1:EFI \
                --new=2:0:+"${DISK_SWAP_SIZE_GB}"GiB --typecode=2:8200 --change-name=2:swap \
                --new=3:0:0 --typecode=3:8300 --change-name=3:pool0_0 \
                "${DISK}"
EFI_PARTITION=1
SWAP_PARTITION=2
CRYPT_PARTITION=3

Wipe and Partition Disk (Swap File)

wipefs "${DISK}" -a -f
sgdisk --zap-all "${DISK}"
sgdisk --clear \
                --new=1:0:+"${DISK_EFI_SIZE_MB}"MiB --typecode=1:ef00 --change-name=1:EFI \
                --new=2:0:0 --typecode=2:8200 --change-name=2:pool0_0 \
                "${DISK}"
EFI_PARTITION=1
CRYPT_PARTITION=2

Create Encrypted Disk and Initial Filesystems

echo "${PASSWORD}" | cryptsetup --verify-passphrase -v luksFormat "${DISK}""${CRYPT_PARTITION}"
echo "${PASSWORD}" | cryptsetup open "${DISK}""${CRYPT_PARTITION}" pool0_0
mkfs.btrfs -f /dev/mapper/pool0_0
RAID1
echo "${PASSWORD}" | cryptsetup --verify-passphrase -v luksFormat "${DISK}""${CRYPT_PARTITION}"
echo "${PASSWORD}" | cryptsetup open "${DISK}""${CRYPT_PARTITION}" pool0_0
echo "${PASSWORD}" | cryptsetup --verify-passphrase -v luksFormat "${DISK2}"
echo "${PASSWORD}" | cryptsetup open "${DISK2}" pool0_1
mkfs.btrfs -f -m raid1 -d raid1 /dev/mapper/pool0_[01]

Create BTRFS Subvolumes

mount -t btrfs /dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2) /mnt
btrfs subvolume create /mnt/root
mkdir -p /mnt/home
btrfs subvolume create /mnt/home/active
btrfs subvolume create /mnt/home/snapshots
btrfs subvolume create /mnt/nix
mkdir -p /mnt/persist
btrfs subvolume create /mnt/persist/active
btrfs subvolume create /mnt/persist/snapshots
btrfs subvolume create /mnt/swap
mkdir -p /mnt/var_local
btrfs subvolume create /mnt/var_local/active
btrfs subvolume create /mnt/var_local/snapshots
btrfs subvolume create /mnt/var_log

Take a readonly snapshot of the root subvolume, which will get rolled back to on every boot.

btrfs subvolume snapshot -r /mnt/root /mnt/root-blank

Mount Subvolumes and Partitions

umount /mnt
mount -o subvol=root,compress=zstd,noatime /dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2) /mnt

mkdir -p /mnt/home
mount -o subvol=home/active,compress=zstd,noatime /dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2) /mnt/home
mkdir -p /mnt/home/.snapshots
mount -o subvol=home/snapshots,compress=zstd,noatime /dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2) /mnt/home/.snapshots

mkdir -p /mnt/nix
mount -o subvol=nix,compress=zstd,noatime /dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2) /mnt/nix

mkdir -p /mnt/persist
mount -o subvol=persist/active,compress=zstd,noatime /dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2) /mnt/persist
mkdir -p /mnt/persist/.snapshots
mount -o subvol=persist/snapshots,compress=zstd,noatime /dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2) /mnt/persist


mkdir -p /mnt/var/local
mount -o subvol=var_local/active,compress=zstd,noatime /dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2) /mnt/var/local
mkdir -p /mnt/var/local/.snapshots
mount -o subvol=var_local/snapshots,compress=zstd,noatime /dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2) /mnt/var/local/.snapshots

mkdir -p /mnt/var/log
mount -o subvol=var_log,compress=zstd,noatime /dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2) /mnt/var/log

mkdir -p /mnt/boot
mount -o defaults,nosuid,nodev,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro /dev/disk/by-partlabel/EFI /mnt/boot/

Setup Swap

Parition
mkswap "${DISK}""${SWAP_PARTITION}"
swapon
File Based Swap
  • You can only use File Based swap on non RAID drives.
mkdir -p /mnt/swap
mount -o subvol=swap,compress=zstd,noatime /dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2) /mnt/swap
mkdir -p /mnt/swap
touch /mnt/swap/swapfile
chattr +C /mnt/swap/swapfile
dd if=/dev/zero of=mnt/swap/swapfile bs=1024 count=0 seek=$(expr "$(cat /proc/meminfo | grep "MemTotal:" | awk '{$2=$2/(1024^2); print $2;}' | awk '{print ($0-int($0)>0)?int($0)+1:int($0)}')" + 2))G
mkswap /mnt/swap/swapfile 
chmod 600 /mnt/swap/swapfile
swapon /mnt/swap/swapfile
sudo dd if=/dev/zero of=/swapfile bs=1024 count=1048576

# Other way of forcing it in

sed -i '/boot = {/i \
  ## Swap File \
  swapDevices = [{\
    device = \"\/swap\/swapfile\";\
    size = (1024 * 16) + (1024 * 2); # RAM size + 2 GB\
  }];\
  fileSystems."/wap".options = [ "subvol=swap" "compress=none" "noatime"  ];\
  ' /mnt/etc/nixos/configuration.nix
OR 

sed -i '1h;1!H;$!d;x;s/.*fileSystems.*};\n[^\n]*/&   \
  swapDevices = [{\
    device = \"\/swap\/swapfile\";\
    size = $(expr "$(cat /proc/meminfo | grep "MemTotal:" | awk '{$2=$2/(1024^2); print $2;}' | awk '{print ($0-int($0)>0)?int($0)+1:int($0)}')" + 2)); # RAM size + 2 GB\
  }];\n/' /mnt/etc/nixos/hardware-configuration.nix

## Hibernation
sed -i '/boot = {/i \
  boot.kernelParams = [ \"resume_offset=$(btrfs inspect-internal map-swapfile -r /mnt/swap/swapfile)\" ];\n\
  boot.resumeDevice = \"/dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2)\" ;"  /mnt/etc/nixos/configuration.nix

Generate NixOS Hardware Configuration

nixos-generate-config --root /mnt

Force BTRFS mount options

Nixos-generate config isn't smart enough to add options as per here.

sed -i '/boot = {/i \
  # Set Filesystms to noatime and zstd compression
  fileSystems."/".options = [ "subvol=root" "compress=zstd" "noatime"  ]; \
  fileSystems."/home".options = [ "subvol=home/active" "compress=zstd" "noatime"  ]; \
  fileSystems."/home/.snapshots".options = [ "subvol=home/snapshot" "compress=zstd" "noatime"  ]; \
  fileSystems."/nix".options = [ "subvol=nix" "compress=zstd" "noatime"  ]; \
  fileSystems."/persist".options = [ "subvol=persist/active" "compress=zstd" "noatime"  ]; \
  filesystems."/persist".neededForBoot = true ; \  
  fileSystems."/persist/.snapshots".options = [ "subvol=persist/snapshots" "compress=zstd" "noatime"  ]; \
  fileSystems."/var/local".options = [ "subvol=var_local/active" "compress=zstd" "noatime"  ]; \
  fileSystems."/var/local/.snapshots".options = [ "subvol=var_local/snapshot" "compress=zstd" "noatime"  ]; \
  fileSystems."/var/log".options = [ "subvol=var_log" "compress=zstd" "noatime"  ]; \
  filesystems."/var/log".neededForBoot = true ; \
  \n' /mnt/etc/nixos/configuration.nix

Allow discards to encrypted root

sed -i '/boot = {/a\  boot.initrd.luks.devices."pool0_0".allowDiscards = true;' /mnt/etc/nixos/configuration.nix
Raid1
sed -i '/boot = {/a\  boot.initrd.luks.devices."pool0_0".allowDiscards = true;' /mnt/etc/nixos/configuration.nix 
sed -i '/boot = {/a\  boot.initrd.luks.devices."pool0_1".allowDiscards = true;' /mnt/etc/nixos/configuration.nix

Configure Install

Choose either the configuration.nix track or Nix Flakes

configuration.nix

/etc/nixos/configuration.nix

{ config, pkgs, ... }:

{
  imports =
    [ 
      ./hardware-configuration.nix
    ];

  boot = {
    loader = {
      efi = {
        canTouchEfiVariables = false;
      };
      grub = {
        enable = true;
        device = "nodev";  # No device for EFI
        efiSupport = true;
        enableCryptodisk = false;
        useOSProber = false;
        efiInstallAsRemovable = true; # in case canTouchEfiVariables doesn't work for your system
      };
    };

    supportedFilesystems = [ 
      "btrfs"
      "fat" "vfat" "exfat" "ntfs" # Microsoft 
      "cifs"                      # Windows Network Share
    ];
  };

  fileSystems."/".options = [ "subvol=root" "compress=zstd" "noatime"  ];
  fileSystems."/home".options = [ "subvol=home/active" "compress=zstd" "noatime"  ];
  fileSystems."/home/.snapshots".options = [ "subvol=home/snapshot" "compress=zstd" "noatime"  ];
  fileSystems."/nix".options = [ "subvol=nix" "compress=zstd" "noatime"  ];
  fileSystems."/persist".options = [ "subvol=persist/active" "compress=zstd" "noatime"  ];
  filesystems."/persist".neededForBoot = true ;
  fileSystems."/persist/.snapshots".options = [ "subvol=persist/snapshots" "compress=zstd" "noatime"  ];
  fileSystems."/var/local".options = [ "subvol=var_local/active" "compress=zstd" "noatime"  ];
  fileSystems."/var/local/.snapshots".options = [ "subvol=var_local/snapshot" "compress=zstd" "noatime"  ];
  fileSystems."/var/log".options = [ "subvol=var_log" "compress=zstd" "noatime"  ];
  filesystems."/var/log".neededForBoot = true ;

  networking = {
    hostName = "beef"; 
    networkmanager.enable = true;
  };

  i18n.defaultLocale = "en_US.UTF-8"; 

  security.sudo.extraConfig = ''
    Defaults lecture = never
  '';
  
  services = {
    cinnamon.apps.enable = false ;
    btrbk = {
      instances."btrbak" = {
        onCalendar = "*-*-* *:00:00";
        settings = {
          timestamp_format = "long";
          snapshot_preserve_min = "2d";
          preserve_day_of_week = "sunday" ;
          preserve_hour_of_day = "0" ;
          target_preserve = "48h 10d 4w 12m 10y" ;
          volume."/home" = {
            snapshot_create = "always";
            subvolume = ".";
            snapshot_dir = ".snapshots";
          };
          volume."/var/local" = {
            snapshot_create = "always";
            subvolume = ".";
            snapshot_dir = ".snapshots";
          };
        };  
      };
    };
    
    openssh = {
      enable = true;
      settings = {
        PermitRootLogin = "yes" ;
      };
    };
    xserver = { 
      enable = true;
      layout = "us";
      libinput.enable = true;

      displayManager.gdm.enable = true ;
      desktopManager.cinnamon.enable = true;
    };
  };

  system.stateVersion = "23.05";

  systemd.services = {
    create-swapfile = {
      serviceConfig.Type = "oneshot";
      wantedBy = [ "swap-swapfile.swap" ];
      script = ''
        swapfile="/swap/swapfile"
        if [ -f "$swapfile" ]; then
            echo "Swap file $swapfile already exists, taking no action"
        else
            echo "Setting up swap file $swapfile"
            ${pkgs.coreutils}/bin/truncate -s 0 "$swapfile"
            ${pkgs.e2fsprogs}/bin/chattr +C "$swapfile"
        fi
      '';
    };
  };
  

  time.timeZone = "America/Vancouver";

  users.users.dave = {
     description = "Dave Conroy";
     createHome = true ;
     home = "/home/dave" ;
     shell = "/bin/sh" ; # This is actually bash
     group = "users" ;
     extraGroups = [ "wheel" "docker" ];
     uid = 2323;
     isNormalUser = true;
     hashedPassword = "$y$j9T$jEYLXjGrR06/tp76fxyDq/$mX4GTWL7CjVXgAcS5nAHEiT6WIH8uD/IfXj16fuTRQ1";
     packages = with pkgs; [
     ];
   };

}

Flakes

Create Flake

{
  description = "Tired of I.T! NixOS configuration";  

  inputs = {
    nixpkgs.url = "github:nixos/nixpkgs/nixos-23.05";
    unstable.url = "github:NixOS/nixpkgs/nixos-unstable";

    impermanence.url = "github:nix-community/impermanence";
  };  
  
  outputs = { nixpkgs ... }: {
    nixosConfigurations = {
      beef = nixpkgs.lib.nixosSystem {
        system = "x86_64-linux";
        modules = [
          ({ config, pkgs, ... }:
              imports =
                [
                  ./hardware-configuration.nix
                  ./configuration.nix
                ];
            }
          )
        ];
      };
    };
  };
}

Install

configuration.nix

nixos-install

Flake

Need to fetch Nix Flakes and git to clone your repository

cp -R /mnt/etc/nixos /tmp/
nix-shell -p git nixFlakes
git clone https://github.com/tiredofit/sys.git /mnt/etc/nixos

Need to then activate the flake. The trailer of the line is the hostname

nixos-install --root /mnt --flake /mnt/etc/nixos#beef

Configuring Impermanence

Boot into new system make any changes. Afterwords, run script below to detect changes in snapshot vs root since boot and start working on maping files that need to be available upon each reboot.

Create differences script

#!/usr/bin/env bash

_tmp_root=$(mktemp -d)
mkdir -p "${_tmp_root}"
mount -o subvol=/ /dev/mapper/pool0_0 "${_tmp_root}" > /dev/null 2>&1

set -euo pipefail

OLD_TRANSID=$(sudo btrfs subvolume find-new ${_tmp_root}/root-blank 9999999)
OLD_TRANSID=${OLD_TRANSID#transid marker was }

sudo btrfs subvolume find-new "${_tmp_root}/root" "$OLD_TRANSID" | sed '$d' | cut -f17- -d' ' | sort | uniq | 
while read path; do
    path="/$path"
    if [ -L "$path" ]; then
        : # The path is a symbolic link, so is probably handled by NixOS already
    elif [ -d "$path" ]; then
        : # The path is a directory, ignore
    else
        echo "$path"
    fi
done
umount "${_tmp_root}"
rm -rf "${_tmp_root}"

Create Configuration to reset root on each reboot

  boot.initrd.postDeviceCommands = pkgs.lib.mkBefore ''
    mkdir -p /mnt
    mount -o subvol=/ /dev/mapper/pool0_0 /mnt
    btrfs subvolume list -o /mnt/root | cut -f9 -d' ' |
    while read subvolume; do
        echo "Deleting /$subvolume subvolume"
        btrfs subvolume delete "/mnt/$subvolume"
    done &&
    echo "Deleting /root subvolume" &&
    btrfs subvolume delete /mnt/root
    echo "Restoring blank /root subvolume"
    btrfs subvolume snapshot /mnt/root-blank /mnt/root
    umount /mnt
  '';
RAID1
  • Instead of relying on postDeviceCommands you can create a systemD service in initrd. This could also be used on single drive as well.
    boot.initrd.systemd = {
      enable = true;
      services.rollback = {
        description = "Rollback BTRFS root subvolume to a pristine state";
        wantedBy = [
          "initrd.target"
        ];
        after = [
          "systemd-cryptsetup@pool0_0.service"
        ];
        before = [
          "sysroot.mount"
        ];
        unitConfig.DefaultDependencies = "no";
        serviceConfig.Type = "oneshot";
        script = ''
          mkdir -p /mnt
          mount -o subvol=/ /dev/mapper/pool0_0 /mnt
          btrfs subvolume list -o /mnt/root | cut -f9 -d' ' |
          while read subvolume; do
            echo "Deleting /$subvolume subvolume"
            btrfs subvolume delete "/mnt/$subvolume"
          done &&
          echo "Deleting /root subvolume" &&
          btrfs subvolume delete /mnt/root
          echo "Restoring blank /root subvolume"
          btrfs subvolume snapshot /mnt/root-blank /mnt/root
          umount /mnt
        '';
      };
    };

Configuring Impermanence

{ config, pkgs, ... }:

let
  impermanence = builtins.fetchTarball "https://github.com/nix-community/impermanence/archive/master.tar.gz";
in
{
  imports = [ "${impermanence}/nixos.nix" ];

  # this folder is where the files will be stored (don't put it in tmpfs)
  environment.persistence."/persist" = { 
    hideMounts = true ; 
    directories = [
      "/etc/nixos"                       # NixOS
      "/etc/NetworkManager"              # NetworkManager
      { directory = "/var/lib/colord"; 
        user = "colord"; 
        group = "colord"; 
        mode = "u=rwx,g=rx,o="; }        # Colord Profiles
      "/var/lib/bluetooth"               # Bluetooth
      "/var/lib/docker"                  # Docker
      
    ];
    files = [   
      "/etc/ssh/ssh_host_*"                 # SSH
      "/etc/machine-id"                     # Needed for SystemD Journal
      "/var/lib/NetworkManager/secret_key"  # Network Manager
      "/var/lib/NetworkManager/seen-bssids" # Network Manager
      "/var/lib/NetworkManager/timestamps"  # Network Manager
      { file = "/etc/nix/id_rsa"; parentDirectory = { mode = "u=rwx,g=,o="; }; } # Nix
    ];
  };cnn.com

Persistence Rules

Bluetooth

  systemd.tmpfiles.rules = [
    "L /var/lib/bluetooth - - - - /persist/var/lib/bluetooth"
  ];

##
sudo mkdir -p /persist/var/lib/bluetooth

Docker

  virtualisation = {
     docker.enable = true;
  };
  
    systemd.tmpfiles.rules = [
    "L /var/lib/docker - - - - /persist/var/lib/docker"
  ];

##
sudo systemctl stop docker
sudo mkdir -p /persist/var/lib/
sudo cp -r {,/persist}/var/lib/docker

Network Manager

  environment.etc."NetworkManager/system-connections" = {
    source = "/persist/etc/NetworkManager/system-connections/";
  };

  systemd.tmpfiles.rules = [
    "L /var/lib/NetworkManager/secret_key - - - - /persist/var/lib/NetworkManager/secret_key"
    "L /var/lib/NetworkManager/seen-bssids - - - - /persist/var/lib/NetworkManager/seen-bssids"
    "L /var/lib/NetworkManager/timestamps - - - - /persist/var/lib/NetworkManager/timestamps"
  ];
  
##
sudo mkdir -p /persist/etc/NetworkManager
sudo cp -r {,/persist}/etc/NetworkManager/system-connections
sudo mkdir -p /persist/var/lib/NetworkManager
sudo cp /var/lib/NetworkManager/{secret_key,seen-bssids,timestamps} /persist/var/lib/NetworkManager/

NixOS

 environment.etc = {
    nixos.source = "/persist/etc/nixos";
    NIXOS.source = "/persist/etc/NIXOS";
    machine-id.source = "/persist/etc/machine-id";
  };
    
sudo cp -R {,/persist}/etc/nixos
sudo cp -R {,/persist}/etc/NIXOS
sudo cp {,/persist}/etc/machine-id

OpenSSH

  services.openssh = {
    enable = true;
    hostKeys = [
      {
        path = "/persist/etc/ssh/ssh_host_ed25519_key";
        type = "ed25519";
      }
      {
        path = "/persist/etc/ssh/ssh_host_rsa_key";
        type = "rsa";
        bits = 4096;
      }
    ];
  };

##
sudo mkdir -p /persist/etc/ssh
sudo cp -R /etc/ssh/ssh_host* /persist/etc/ssh/

Keeping up to date

Normal

sudo nix-channel --update
sudo nixos-rebuild switch

Flakes

sudo nix flake update /etc/nixos/
sudo nixos-rebuild switch --flake /etc/nixos/#beef

Scratchpad

Unmount commands

umount /mnt/boot
umount /mnt/home/.snapshots
umount /mnt/home
umount /mnt/nix
umount /mnt/persist/.snapshots
umount /mnt/persist
umount /mnt/var/local/.snapshots
umount /mnt/var/local
umount /mnt/var/log
umount /mnt

Restart Install over and over

umount /mnt/boot
umount /mnt/home/.snapshots
umount /mnt/home
umount /mnt/nix
umount /mnt/persist/.snapshots
umount /mnt/persist
umount /mnt/swap
umount /mnt/var/local/.snapshots
umount /mnt/var/local
umount /mnt/var/log
umount /mnt/boot/efi
umount /mnt

PASSWORD=password
DISK_SWAP_SIZE_GB=4
DISK_EFI_SIZE_MB=512
DISK=/dev/vda

## Swap Partition
wipefs "${DISK}" -a -f
sgdisk --zap-all "${DISK}"
sgdisk --clear \
                --new=1:0:+"${DISK_EFI_SIZE_MB}"MiB --typecode=1:ef00 --change-name=1:EFI \
                --new=2:0:+"${DISK_SWAP_SIZE_GB}"GiB --typecode=2:8200 --change-name=2:swap \
                --new=3:0:0 --typecode=3:8300 --change-name=3:pool0_0 \
                "${DISK}"
EFI_PARTITION=1
SWAP_PARTITION=2
CRYPT_PARTITION=3

## Swap File
wipefs "${DISK}" -a -f
sgdisk --zap-all "${DISK}"
sgdisk --clear \
                --new=1:0:+"${DISK_EFI_SIZE_MB}"MiB --typecode=1:ef00 --change-name=1:EFI \
                --new=2:0:0 --typecode=2:8300 --change-name=2:pool0_0 \
                "${DISK}"
EFI_PARTITION=1
CRYPT_PARTITION=2
echo "${PASSWORD}" | cryptsetup luksFormat "${DISK}""${CRYPT_PARTITION}"
echo "${PASSWORD}" | cryptsetup open "${DISK}""${CRYPT_PARTITION}" pool0_0
mkfs.btrfs -f /dev/mapper/pool0_0
mkfs.vfat "${DISK}""${EFI_PARTITION}"
mount -t btrfs /dev/mapper/pool0_0 /mnt
btrfs subvolume create /mnt/root
mkdir -p /mnt/home
btrfs subvolume create /mnt/home/active
btrfs subvolume create /mnt/home/snapshots
btrfs subvolume create /mnt/nix
btrfs subvolume create /mnt/persist
btrfs subvolume create /mnt/swap
mkdir -p /mnt/var_local
btrfs subvolume create /mnt/var_local/active
btrfs subvolume create /mnt/var_local/snapshots
btrfs subvolume create /mnt/var_log
btrfs subvolume snapshot -r /mnt/root /mnt/root-blank
umount /mnt
mount -o subvol=root,compress=zstd,noatime /dev/mapper/pool0_0 /mnt
mkdir -p /mnt/home
mount -o subvol=home/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home
mkdir -p /mnt/home/.snapshots
mount -o subvol=home/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home/.snapshots
mkdir -p /mnt/nix
mount -o subvol=nix,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/nix
mkdir -p /mnt/persist
mount -o subvol=persist/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/persist
mkdir -p /mnt/persist/.snapshots
mount -o subvol=persist/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/persist/.snapshots
mkdir -p /mnt/swap
mount -o subvol=swap,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/swap
mkdir -p /mnt/var/local
mount -o subvol=var_local/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local
mkdir -p /mnt/var/local/.snapshots
mount -o subvol=var_local/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local/.snapshots
mkdir -p /mnt/boot
mount -o defaults,nosuid,nodev,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro /dev/disk/by-partlabel/EFI /mnt/boot


mkdir -p /mnt/swap
touch /mnt/swap/swapfile
chattr +C /mnt/swap/swapfile
dd if=/dev/zero of=mnt/swap/swapfile bs=1024 count=0 seek=1G
mkswap /mnt/swap/swapfile 
chmod 600 /mnt/swap/swapfile
swapon /mnt/swap/swapfile
sudo dd if=/dev/zero of=/swapfile bs=1024 count=1048576


mkswap "${SWAP_PARTITION}
swapon "${SWAP_PARTITION}"


nixos-generate-config --root /mnt

sed -i '/boot = {/i \
  # Set Filesystms to noatime and zstd compression
  fileSystems."/".options = [ "subvol=root" "compress=zstd" "noatime"  ]; \
  fileSystems."/home".options = [ "subvol=home/active" "compress=zstd" "noatime"  ]; \
  fileSystems."/home/.snapshots".options = [ "subvol=home/snapshot" "compress=zstd" "noatime"  ]; \
  fileSystems."/nix".options = [ "subvol=nix" "compress=zstd" "noatime"  ]; \
  fileSystems."/persist".options = [ "subvol=persist" "compress=zstd" "noatime"  ]; \
  filesystems."/persist".neededForBoot = true ; \  
  fileSystems."/var/local".options = [ "subvol=var_local/active" "compress=zstd" "noatime"  ]; \
  fileSystems."/var/local/.snapshots".options = [ "subvol=var_local/snapshot" "compress=zstd" "noatime"  ]; \
  fileSystems."/var/log".options = [ "subvol=var_log" "compress=zstd" "noatime"  ]; \
  filesystems."/var/log".neededForBoot = true ; \
  \n' /mnt/etc/nixos/configuration.nix  
  
  rm -rf /mnt/etc/nixos/configuration.nix
nano /mnt/etc/nixos/configuration.nix

Full configuration.nix

{ config, lib, modulesPath, pkgs, ... }:

let
  impermanence = builtins.fetchTarball "https://github.com/nix-community/impermanence/archive/master.tar.gz";
in

{
  imports = [
    (modulesPath + "/profiles/qemu-guest.nix")
    "${impermanence}/nixos.nix"
  ];

  boot = {
    extraModulePackages = [ ];
    extraFiles = {}
    initrd = {
      availableKernelModules = [ "ahci" "xhci_pci" "virtio_pci" "virtio_scsi" "sr_mod" "virtio_blk" ];

      luks = {
        devices = {
          "pool0_0" = {
             allowDiscards = true;
             bypassWorkqueues = true;
             device = "/dev/disk/by-uuid/2e2cef00-ed7a-41fa-b173-b1ff9bdcbfeb";
          };
          "pool0_1" = {
             allowDiscards = true;
             bypassWorkqueues = true;
             device = "/dev/disk/by-uuid/722ec824-876d-471b-ae30-5107f1e8264d";
          };
        };
      };

      supportedFilesystems = [
        "btrfs"
        "vfat"
      ];
    };

    systemd = {
      enable = true;
      services.rollback = {
        description = "Rollback BTRFS root subvolume to a pristine state";
        wantedBy = [
          "initrd.target"
        ];
        after = [
          "systemd-cryptsetup@enc.service"
        ];
        before = [
          "sysroot.mount"
        ];
        unitConfig.DefaultDependencies = "no";
        serviceConfig.Type = "oneshot";
        script = ''
          mkdir -p /mnt
          mount -o subvol=/ /dev/mapper/pool0_0 /mnt
          btrfs subvolume list -o /mnt/root | cut -f9 -d' ' |
          while read subvolume; do
            echo "Deleting /$subvolume subvolume"
            btrfs subvolume delete "/mnt/$subvolume"
          done &&
          echo "Deleting /root subvolume" &&
          btrfs subvolume delete /mnt/root
          echo "Restoring blank /root subvolume"
          btrfs subvolume snapshot /mnt/root-blank /mnt/root
          umount /mnt
        '';
      };
      services.persisted-files = {
        description = "Hard-link persisted files from /persist";
        wantedBy = [
          "initrd.target"
        ];
        after = [
          "sysroot.mount"
        ];
        unitConfig.DefaultDependencies = "no";
        serviceConfig.Type = "oneshot";
        script = ''
          mkdir -p /sysroot/etc/
          ln -snfT /persist/etc/machine-id /sysroot/etc/machine-id
        '';
      };
    };

    kernelModules = [ ];

    loader = {
      efi = {
        canTouchEfiVariables = false;
      };
      grub = {
        enable = true;
        device = "nodev";
        efiSupport = true;
        enableCryptodisk = false;
        useOSProber = false;
        efiInstallAsRemovable = true;
      };
    };
  };

  environment = {
    persistence = {
      "/persist" = {
         hideMounts = true ;
         directories = [
           "/etc/nixos"
           "/etc/NetworkManager"
           "/root"
           "/var/lib/NetworkManager"
         ];
      };
    };
    systemPackages = with pkgs; [
      curl
      nano
      wget
    ];
  };

  fileSystems = {
    "/" = {
      device = "/dev/disk/by-uuid/1b864b78-f28a-4730-8efd-759b895b567e";
      fsType = "btrfs";
      options = [ "subvol=root" "compress=zstd" "noatime" ];
    };

    "/boot" = {
      device = "/dev/disk/by-uuid/6DD4-987C";
      fsType = "vfat";
    };

    "/home" = {
      device = "/dev/disk/by-uuid/1b864b78-f28a-4730-8efd-759b895b567e";
      fsType = "btrfs";
      neededForBoot = true;
      options = [ "subvol=home/active" "compress=zstd" "noatime" ];
    };

    "/home/.snapshots" = {
      device = "/dev/disk/by-uuid/1b864b78-f28a-4730-8efd-759b895b567e";
      fsType = "btrfs";
      options = [ "subvol=home/snapshots" "compress=zstd" "noatime" ];
    };

    "/nix" = {
      device = "/dev/disk/by-uuid/1b864b78-f28a-4730-8efd-759b895b567e";
      fsType = "btrfs";
      options = [ "subvol=nix" "compress=zstd" "noatime" ];
    };

    "/persist" = {
      device = "/dev/disk/by-uuid/1b864b78-f28a-4730-8efd-759b895b567e";
      fsType = "btrfs";
      neededForBoot = true;
      options = [ "subvol=persist/active" "compress=zstd" "noatime" ];
    };

    "/persist" = {
      device = "/dev/disk/by-uuid/1b864b78-f28a-4730-8efd-759b895b567e";
      fsType = "btrfs";
      neededForBoot = true;
      options = [ "subvol=persist/snapshots" "compress=zstd" "noatime" ];
    };

    "/var/local" = {
      device = "/dev/disk/by-uuid/1b864b78-f28a-4730-8efd-759b895b567e";
      fsType = "btrfs";
      options = [ "subvol=var_local/active" "compress=zstd" "noatime" ];
    };

    "/var/local/.snapshots" = {
      device = "/dev/disk/by-uuid/1b864b78-f28a-4730-8efd-759b895b567e";
      fsType = "btrfs";
      options = [ "subvol=var_local/snapshots" "compress=zstd" "noatime" ];
    };

    "/var/log" = {
      device = "/dev/disk/by-uuid/1b864b78-f28a-4730-8efd-759b895b567e";
      fsType = "btrfs";
      neededForBoot = true;
      options = [ "subvol=var_log" "compress=zstd" "noatime" ];
    };
  };

  networking = {
    hostName = "nixos";
    networkmanager.enable = true ;
    useDHCP = lib.mkDefault true;
  };

  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";

  security.sudo = {
    extraConfig = ''
      Defaults lecture = never
    '';
    wheelNeedsPassword = false ;
  };

  services.openssh.enable = true;

  swapDevices = [
    { device = "/dev/disk/by-uuid/c179fdb0-c9c2-42d8-bf12-8da81fdbef30"; }
  ];

  system.stateVersion = "23.05";

  time.timeZone = "America/Vancouver";

  users.users.dave = {
    createHome = true ;
    description = "Dave";
    extraGroups = [ "wheel" ];
    group = "users" ;
    hashedPassword = "hashed password";
    home = "/home/dave" ;
    isNormalUser = true;
    shell = "/bin/sh" ;
    uid = 2323;
  };
}

Setup Impermanance

   nixos-rebuild boot
   sudo mkdir -p /persist/var/lib/
   sudo cp -r {,/persist}/var/lib/docker
   nano /etc/nixos/configuration.nix 
   sudo mkdir -p /persist/etc/NetworkManager
   sudo cp -r {,/persist}/etc/NetworkManager/system-connections
   sudo mkdir -p /persist/var/lib/NetworkManager
   sudo cp /var/lib/NetworkManager/{secret_key,seen-bssids,timestamps} /persist/var/lib/NetworkManager/  
   sudo mkdir -p /persist/etc/ssh
   sudo cp -R /etc/ssh/ssh_host* /persist/etc/ssh/
   sudo cp -R {,/persist}/etc/nixos
   sudo cp -R {,/persist}/etc/NIXOS
   sudo cp {,/persist}/etc/machine-id
   reboot

Errors

Encrypted /boot

When mounting the ESP partition at /boot/efi instead of /boot grub picks the mounted pool0_0 filesystem and puts it in its configuration. Idea - Creating a seperate LUKS boot partition??

Back to top