Skip to main content

Installing NixOS (Encrypted BTRFS Impermanance)

Intro

This details the steps to create a LUKS encrypted BTRFS NixOS installation with impermanance, only keeping the settings that are required in between reboots.

Features:

  • UEFI Boot w/GRUB
  • Encrypted Btrfs File System with hourly snapshots on /home and /var/local
  • Swap File instead of Swap Partition w/Hibernate Support

This is a work in progress using the standard configuration.nix setup.

Todo
  • Setup FDE instead of Partition 2 Errors

Disk Setup

Setup Variables

DISK_EFI_SIZE_MB=512
DISK=/dev/vda
PASSWORD=password

Wipe and Partition Disk (Swap File)

wipefs "${DISK}" -a -f
sgdisk --zap-all "${DISK}"
sgdisk --clear \
                --new=1:0:+"${DISK_EFI_SIZE_MB}"MiB --typecode=1:ef00 --change-name=1:EFI \
                --new=2:0:0 --typecode=2:8200 --change-name=2:pool0_0 \
                "${DISK}"
EFI_PARTITION=1
CRYPT_PARTITION=2

Create Encrypted Disk and Initial Filesystems

echo "${PASSWORD}" | cryptsetup --verify-passphrase -v luksFormat "${CRYPT_PARTITION}"
echo "${PASSWORD}" | cryptsetup open "${DISK}""${CRYPT_PARTITION}" pool0_0
mkfs.btrfs -f /dev/mapper/pool0_0
mkfs.vfat "${DISK}""${EFI_PARTITION}"

Create BTRFS Subvolumes

mount -t btrfs /dev/mapper/pool0_0 /mnt
btrfs subvolume create /mnt/root
mkdir -p /mnt/home
btrfs subvolume create /mnt/home/active
btrfs subvolume create /mnt/home/snapshots
btrfs subvolume create /mnt/nix
btrfs subvolume create /mnt/persist
btrfs subvolume create /mnt/swap
mkdir -p /mnt/var_local
btrfs subvolume create /mnt/var_local/active
btrfs subvolume create /mnt/var_local/snapshots
btrfs subvolume create /mnt/var_log

Take a readonly snapshot of the root subvolume, which will get rolled back to on every boot.

btrfs subvolume snapshot -r /mnt/root /mnt/root-blank

Mount Subvolumes and Partitions

umount /mnt
mount -o subvol=root,compress=zstd,noatime /dev/mapper/pool0_0 /mnt

mkdir -p /mnt/home
mount -o subvol=home/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home
mkdir -p /mnt/home/.snapshots
mount -o subvol=home/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home/.snapshots

mkdir -p /mnt/nix
mount -o subvol=nix,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/nix

mkdir -p /mnt/persist
mount -o subvol=persist,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/persist

mkdir -p /mnt/swap
mount -o subvol=swap,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/swap

mkdir -p /mnt/var/local
mount -o subvol=var_local/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local
mkdir -p /mnt/var/local/.snapshots
mount -o subvol=var_local/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local/.snapshots

mkdir -p /mnt/var/log
mount -o subvol=var_log,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/log

mkdir -p /mnt/boot/efi
mount -o defaults,nosuid,nodev,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro /dev/disk/by-partlabel/EFI /mnt/boot/efi

Generate NixOS Hardware Configuration

nixos-generate-config --root /mnt

File Based Swap

Use a partition on RAID

sed -i '1h;1!H;$!d;x;s/.*fileSystems.*};\n[^\n]*/&   \
  swapDevices = [{\
    device = \"\/swap\/swapfile\";\
    size = (1024 * 16) + (1024 * 2); # RAM size + 2 GB\n  }];\n/' /mnt/etc/nixos/hardware-configuration.nix

sed -i '/swapDevices = \[ \];/d' /mnt/etc/nixos/hardware-configuration.nix

truncate -s 0 /mnt/swap/swapfile
chattr +C /mnt/swap/swapfile

sed -i "/boot.extraModulePackages/a\\
  boot.kernelParams = [ \"resume_offset=$(btrfs inspect-internal map-swapfile -r /mnt/swap/swapfile)\" ];\n\
  boot.resumeDevice = \"/dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2)\" ;"  /mnt/etc/nixos/hardware-configuration.nix

Force var_log and persist subvolume to be available for boot

sed -i "/subvol=var_log/a\      neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix 
sed -i "/subvol=persist/a\      neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix 

Force BTRFS mount options

Nixos-generate config isn't smart enough to add options as per here.

sed -i "s|options = \[ \(.*\) \];|options = \[ \1 \"compress=zstd\" \"noatime\"  \];|g" /mnt/etc/nixos/hardware-configuration.nix 
sed -i 's|"subvol=swap" "compress=zstd"|"subvol=swap" "compress=none"|g' /mnt/etc/nixos/hardware-configuration.nix 

Allow discards to encrypted root

sed -i '/boot.initrd.luks.devices/a\  boot.initrd.luks.devices."pool0_0".allowDiscards = true;' /mnt/etc/nixos/hardware-configuration.nix 

Configure Install

Choose either the configuration.nix track or Nix Flakes

configuration.nix

/etc/nixos/configuration.nix

{ config, pkgs, ... }:

{
  imports =
    [ 
      ./hardware-configuration.nix
    ];

  boot = {
    loader = {
      efi = {
        canTouchEfiVariables = false;
      };
      grub = {
        enable = true;
        device = "nodev";  # No device for EFI
        efiSupport = true;
        enableCryptodisk = false;
        useOSProber = false;
        efiInstallAsRemovable = true; # in case canTouchEfiVariables doesn't work for your system
      };
    };

    supportedFilesystems = [ 
      "btrfs"
      "fat" "vfat" "exfat" "ntfs" # Microsoft 
      "cifs"                      # Windows Network Share
    ];
  };

  networking = {
    hostName = "beef"; 
    networkmanager.enable = true;
  };

  i18n.defaultLocale = "en_US.UTF-8"; 

  security.sudo.extraConfig = ''
    Defaults lecture = never
  '';
  
  services = {
    cinnamon.apps.enable = false ;
    btrbk = {
      instances."btrbak" = {
        onCalendar = "*-*-* *:00:00";
        settings = {
          timestamp_format = "long";
          snapshot_preserve_min = "2d";
          preserve_day_of_week = "sunday" ;
          preserve_hour_of_day = "0" ;
          target_preserve = "48h 10d 4w 12m 10y" ;
          volume."/home" = {
            snapshot_create = "always";
            subvolume = ".";
            snapshot_dir = ".snapshots";
          };
          volume."/var/local" = {
            snapshot_create = "always";
            subvolume = ".";
            snapshot_dir = ".snapshots";
          };
        };  
      };
    };
    
    openssh = {
      enable = true;
      settings = {
        PermitRootLogin = "yes" ;
      };
    };
    xserver = { 
      enable = true;
      layout = "us";
      libinput.enable = true;

      displayManager.gdm.enable = true ;
      desktopManager.cinnamon.enable = true;
    };
  };

  system.stateVersion = "23.05";

  systemd.services = {
    create-swapfile = {
      serviceConfig.Type = "oneshot";
      wantedBy = [ "swap-swapfile.swap" ];
      script = ''
        swapfile="/swap/swapfile"
        if [ -f "$swapfile" ]; then
            echo "Swap file $swapfile already exists, taking no action"
        else
            echo "Setting up swap file $swapfile"
            ${pkgs.coreutils}/bin/truncate -s 0 "$swapfile"
            ${pkgs.e2fsprogs}/bin/chattr +C "$swapfile"
        fi
      '';
    };
  };
  

  time.timeZone = "America/Vancouver";

  users.users.dave = {
     description = "Dave Conroy";
     createHome = true ;
     home = "/home/dave" ;
     shell = "/bin/sh" ; # This is actually bash
     group = "users" ;
     extraGroups = [ "wheel" "docker" ];
     uid = 2323;
     isNormalUser = true;
     hashedPassword = "$y$j9T$jEYLXjGrR06/tp76fxyDq/$mX4GTWL7CjVXgAcS5nAHEiT6WIH8uD/IfXj16fuTRQ1";
     packages = with pkgs; [
     ];
   };

}

Flakes

Create Flake

{
  description = "Tired of I.T! NixOS configuration";  

  inputs = {
    nixpkgs.url = "github:nixos/nixpkgs/nixos-23.05";
    unstable.url = "github:NixOS/nixpkgs/nixos-unstable";

    impermanence.url = "github:nix-community/impermanence";
  };  
  
  outputs = { nixpkgs ... }: {
    nixosConfigurations = {
      nixtstbeef = nixpkgs.lib.nixosSystem {
        system = "x86_64-linux";
        modules = [
          ({ config, pkgs, ... }:
              imports =
                [
                  ./hardware-configuration.nix
                  ./configuration.nix
                ];
            }
          )
        ];
      };
    };
  };
}

Install

configuration.nix

nixos-install 

Flake

Need to fetch Nix Flakes and git to clone your repository

cp -R /mnt/etc/nixos /tmp/
nix-shell -p git nixFlakes
git clone https://github.com/tiredofit/sys.git /mnt/etc/nixos

Need to then activate the flake. The trailer of the line is the hostname

nixos-install --root /mnt --flake /mnt/etc/nixos#beef

s

Configuring Impermanence

Boot into new system make any changes. Afterwords, run script below to detect changes in snapshot vs root since boot and start working on maping files that need to be available upon each reboot.

Create differences script

#!/usr/bin/env bash

_tmp_root=$(mktemp -d)
mkdir -p "${_tmp_root}"
mount -o subvol=/ /dev/mapper/pool0_0 "${_tmp_root}" > /dev/null 2>&1

set -euo pipefail

OLD_TRANSID=$(sudo btrfs subvolume find-new ${_tmp_root}/root-blank 9999999)
OLD_TRANSID=${OLD_TRANSID#transid marker was }

sudo btrfs subvolume find-new "${_tmp_root}/root" "$OLD_TRANSID" | sed '$d' | cut -f17- -d' ' | sort | uniq | 
while read path; do
    path="/$path"
    if [ -L "$path" ]; then
        : # The path is a symbolic link, so is probably handled by NixOS already
    elif [ -d "$path" ]; then
        : # The path is a directory, ignore
    else
        echo "$path"
    fi
done
umount "${_tmp_root}"
rm -rf "${_tmp_root}"

Create Configuration to reset root on each reboot

  boot.initrd.postDeviceCommands = pkgs.lib.mkBefore ''
    mkdir -p /mnt
    mount -o subvol=/ /dev/mapper/pool0_0 /mnt
    btrfs subvolume list -o /mnt/root | cut -f9 -d' ' |
    while read subvolume; do
        echo "Deleting /$subvolume subvolume"
        btrfs subvolume delete "/mnt/$subvolume"
    done &&
    echo "Deleting /root subvolume" &&
    btrfs subvolume delete /mnt/root
    echo "Restoring blank /root subvolume"
    btrfs subvolume snapshot /mnt/root-blank /mnt/root
    umount /mnt
  '';

Configuring Impermanence

{ config, pkgs, ... }:

let
  impermanence = builtins.fetchTarball "https://github.com/nix-community/impermanence/archive/master.tar.gz";
in
{
  imports = [ "${impermanence}/nixos.nix" ];

  # this folder is where the files will be stored (don't put it in tmpfs)
  environment.persistence."/persist" = { 
    hideMounts = true ; 
    directories = [
      "/etc/nixos"                       # NixOS
      "/etc/NetworkManager"              # NetworkManager
      { directory = "/var/lib/colord"; 
        user = "colord"; 
        group = "colord"; 
        mode = "u=rwx,g=rx,o="; }        # Colord Profiles
      "/var/lib/bluetooth"               # Bluetooth
      "/var/lib/docker"                  # Docker
      
    ];
    files = [   
      "/etc/ssh/ssh_host_*"                 # SSH
      "/etc/machine-id"                     # Needed for SystemD Journal
      "/var/lib/NetworkManager/secret_key"  # Network Manager
      "/var/lib/NetworkManager/seen-bssids" # Network Manager
      "/var/lib/NetworkManager/timestamps"  # Network Manager
      { file = "/etc/nix/id_rsa"; parentDirectory = { mode = "u=rwx,g=,o="; }; } # Nix
    ];
  };cnn.com
  


Persistence Rules

Bluetooth

  systemd.tmpfiles.rules = [
    "L /var/lib/bluetooth - - - - /persist/var/lib/bluetooth"
  ];

##
sudo mkdir -p /persist/var/lib/bluetooth

Docker

  virtualisation = {
     docker.enable = true;
  };
  
    systemd.tmpfiles.rules = [
    "L /var/lib/docker - - - - /persist/var/lib/docker"
  ];

##
sudo systemctl stop docker
sudo mkdir -p /persist/var/lib/
sudo cp -r {,/persist}/var/lib/docker

Network Manager

  environment.etc."NetworkManager/system-connections" = {
    source = "/persist/etc/NetworkManager/system-connections/";
  };

  systemd.tmpfiles.rules = [
    "L /var/lib/NetworkManager/secret_key - - - - /persist/var/lib/NetworkManager/secret_key"
    "L /var/lib/NetworkManager/seen-bssids - - - - /persist/var/lib/NetworkManager/seen-bssids"
    "L /var/lib/NetworkManager/timestamps - - - - /persist/var/lib/NetworkManager/timestamps"
  ];
  
##
sudo mkdir -p /persist/etc/NetworkManager
sudo cp -r {,/persist}/etc/NetworkManager/system-connections
sudo mkdir -p /persist/var/lib/NetworkManager
sudo cp /var/lib/NetworkManager/{secret_key,seen-bssids,timestamps} /persist/var/lib/NetworkManager/  

NixOS

 environment.etc = {
    nixos.source = "/persist/etc/nixos";
    NIXOS.source = "/persist/etc/NIXOS";
    machine-id.source = "/persist/etc/machine-id";
  };
    
sudo cp -R {,/persist}/etc/nixos
sudo cp -R {,/persist}/etc/NIXOS
sudo cp {,/persist}/etc/machine-id

OpenSSH

  services.openssh = {
    enable = true;
    hostKeys = [
      {
        path = "/persist/etc/ssh/ssh_host_ed25519_key";
        type = "ed25519";
      }
      {
        path = "/persist/etc/ssh/ssh_host_rsa_key";
        type = "rsa";
        bits = 4096;
      }
    ];
  };

##
sudo mkdir -p /persist/etc/ssh
sudo cp -R /etc/ssh/ssh_host* /persist/etc/ssh/

Keeping up to date

Normal

sudo nix-channel --update
sudo nixos-rebuild switch

Flakes

sudo nix flake update /etc/nixos/
sudo nixnixos-rebuild switch --flake update /etc/nixos/#beef

Scratchpad

Unmount commands

umount /mnt/boot
umount /mnt/home
umount /mnt/nix
umount /mnt/persist
umount /mnt/var/local
umount /mnt/var/log
umount /mnt

Restart Install over and over

umount /mnt/boot
umount /mnt/home/.snapshots
umount /mnt/home
umount /mnt/nix
umount /mnt/persist
umount /mnt/swap
umount /mnt/var/local/.snapshots
umount /mnt/var/local
umount /mnt/var/log
umount /mnt/boot/efi
umount /mnt

PASSWORD=password
DISK_EFI_SIZE_MB=512
DISK=/dev/vda
wipefs "${DISK}" -a -f
sgdisk --zap-all "${DISK}"
sgdisk --clear \
                --new=1:0:+"${DISK_EFI_SIZE_MB}"MiB --typecode=1:ef00 --change-name=1:EFI \
                --new=2:0:0 --typecode=2:8300 --change-name=2:pool0_0 \
                "${DISK}"
EFI_PARTITION=1
CRYPT_PARTITION=2
echo "${PASSWORD}" | cryptsetup luksFormat "${DISK}""${CRYPT_PARTITION}"
echo "${PASSWORD}" | cryptsetup open "${DISK}""${CRYPT_PARTITION}" pool0_0
mkfs.btrfs -f /dev/mapper/pool0_0
mkfs.vfat "${DISK}""${EFI_PARTITION}"
mount -t btrfs /dev/mapper/pool0_0 /mnt
btrfs subvolume create /mnt/root
mkdir -p /mnt/home
btrfs subvolume create /mnt/home/active
btrfs subvolume create /mnt/home/snapshots
btrfs subvolume create /mnt/nix
btrfs subvolume create /mnt/persist
btrfs subvolume create /mnt/swap
mkdir -p /mnt/var_local
btrfs subvolume create /mnt/var_local/active
btrfs subvolume create /mnt/var_local/snapshots
btrfs subvolume create /mnt/var_log
btrfs subvolume snapshot -r /mnt/root /mnt/root-blank
umount /mnt
mount -o subvol=root,compress=zstd,noatime /dev/mapper/pool0_0 /mnt
mkdir -p /mnt/home
mount -o subvol=home/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home
mkdir -p /mnt/home/.snapshots
mount -o subvol=home/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home/.snapshots
mkdir -p /mnt/nix
mount -o subvol=nix,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/nix
mkdir -p /mnt/persist
mount -o subvol=persist,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/persist
mkdir -p /mnt/swap
mount -o subvol=swap,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/swap
mkdir -p /mnt/var/local
mount -o subvol=var_local/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local
mkdir -p /mnt/var/local/.snapshots
mount -o subvol=var_local/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local/.snapshots
mkdir -p /mnt/boot
mount -o defaults,nosuid,nodev,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro /dev/disk/by-partlabel/EFI /mnt/boot
nixos-generate-config --root /mnt

sed -i '1h;1!H;$!d;x;s/.*fileSystems.*};\n[^\n]*/&   \
  swapDevices = [{\
    device = \"\/swap\/swapfile\";\
    size = (1024 * 16) + (1024 * 2); # RAM size + 2 GB\n  }];\n/' /mnt/etc/nixos/hardware-configuration.nix

sed -i '/swapDevices = \[ \];/d' /mnt/etc/nixos/hardware-configuration.nix

truncate -s 0 /mnt/swap/swapfile
chattr +C /mnt/swap/swapfile

sed -i "/boot.extraModulePackages/a\\
  boot.kernelParams = [ \"resume_offset=$(btrfs inspect-internal map-swapfile -r /mnt/swap/swapfile)\" ];\n\
  boot.resumeDevice = \"/dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2)\" ;"  /mnt/etc/nixos/hardware-configuration.nix

sed -i "/subvol=var_log/a\      neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix 
sed -i "/subvol=persist/a\      neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix 
sed -i "s|options = \[ \(.*\) \];|options = \[ \1 \"compress=zstd\" \"noatime\"  \];|g" /mnt/etc/nixos/hardware-configuration.nix 
sed -i 's|"subvol=swap" "compress=zstd"|"subvol=swap" "compress=none"|g' /mnt/etc/nixos/hardware-configuration.nix 
sed -i '/boot.initrd.luks.devices/a\  boot.initrd.luks.devices."pool0_0".allowDiscards = true;' /mnt/etc/nixos/hardware-configuration.nix 
rm -rf /mnt/etc/nixos/configuration.nix
nano /mnt/etc/nixos/configuration.nix

Full configuration.nix

{ config, pkgs, ... }:

let
  impermanence = builtins.fetchTarball "https://github.com/nix-community/impermanence/archive/master.tar.gz";
in
{
  imports =
    [ 
      ./hardware-configuration.nix
      "${impermanence}/nixos.nix"
    ];

 
  boot = {
    initrd.postDeviceCommands = pkgs.lib.mkBefore ''
      mkdir -p /mnt
      mount -o subvol=/ /dev/mapper/pool0_0 /mnt
      btrfs subvolume list -o /mnt/root | cut -f9 -d' ' |
      while read subvolume; do
          echo "Deleting /$subvolume subvolume"
          btrfs subvolume delete "/mnt/$subvolume"
      done &&
      echo "Deleting /root subvolume" &&
      btrfs subvolume delete /mnt/root
      echo "Restoring blank /root subvolume"
      btrfs subvolume snapshot /mnt/root-blank /mnt/root
      umount /mnt
    '';
    loader = {
      efi = {
        canTouchEfiVariables = false;
      };
      grub = {
        enable = true;
        device = "nodev";  # No device for EFI
        efiSupport = true;
        enableCryptodisk = false;
        useOSProber = false;
        efiInstallAsRemovable = true; # in case canTouchEfiVariables doesn't work for your system
      };
      
    };

    kernel.sysctl = {
      "vm.dirty_ratio" = 6;   # sync disk when buffer reach 6% of memory
    };
      
    kernelPackages = pkgs.linuxPackages_latest;  # Latest kernel

    supportedFilesystems = [ 
      "btrfs"
      "fat" "vfat" "exfat" "ntfs" # Microsoft 
      "cifs"                      # Windows Network Share
    ];
  };

  networking = {
    hostName = "beef"; 
    networkmanager.enable = true;
  };

  i18n.defaultLocale = "en_US.UTF-8"; 

  nix = {
    gc = {
      automatic = true;
      dates = "19:00";
      persistent = true;
      options = "--delete-older-than 60d";
    };
  };

  
  nixpkgs.config.allowUnfree = true ; # Allow Non Free packages
  
  security.sudo.extraConfig = ''
    Defaults lecture = never
  '';
  
  services = {
    cinnamon.apps.enable = false ;
    btrbk = {
      instances."btrbak" = {
        onCalendar = "*-*-* *:00:00";
        settings = {
          timestamp_format = "long";
          snapshot_preserve_min = "2d";
          preserve_day_of_week = "sunday" ;
          preserve_hour_of_day = "0" ;
          target_preserve = "48h 10d 4w 12m 10y" ;
          volume."/home" = {
            snapshot_create = "always";
            subvolume = ".";
            snapshot_dir = ".snapshots";
          };
          volume."/var/local" = {
            snapshot_create = "always";
            subvolume = ".";
            snapshot_dir = ".snapshots";
          };
        };
      };
    };
    
    openssh = {
      enable = true;
      settings = {
        PermitRootLogin = "yes" ;
      };
    };
    xserver = { 
      enable = true;
      layout = "us";
      libinput.enable = true;

      displayManager.gdm.enable = true ;
      desktopManager.cinnamon.enable = true;
    };
  };

  # Enable sound.
  sound.enable = true;
  hardware.pulseaudio.enable = true;
  
  system.stateVersion = "23.05";
  systemd.services = {
    create-swapfile = {
      serviceConfig.Type = "oneshot";
      wantedBy = [ "swap-swapfile.swap" ];
      script = ''
        swapfile="/swap/swapfile"
        if [ -f "$swapfile" ]; then
            echo "Swap file $swapfile already exists, taking no action"
        else
            echo "Setting up swap file $swapfile"
            ${pkgs.coreutils}/bin/truncate -s 0 "$swapfile"
            ${pkgs.e2fsprogs}/bin/chattr +C "$swapfile"
        fi
      '';
    };
  };
  

  time.timeZone = "America/Vancouver";

  users.users.dave = {
     description = "Dave Conroy";
     createHome = true ;
     home = "/home/dave" ;
     shell = "/bin/sh" ; # This is actually bash
     group = "users" ;
     extraGroups = [ "wheel" "docker" ];
     uid = 2323;
     isNormalUser = true;
     hashedPassword = "$y$j9T$jEYLXjGrR06/tp76fxyDq/$mX4GTWL7CjVXgAcS5nAHEiT6WIH8uD/IfXj16fuTRQ1";
     packages = with pkgs; [
     ];
   };

  virtualisation = {
     docker.enable = true;
  };
  
  environment.systemPackages = with pkgs; [
      ncdu 
      git 
      rsync
      tmux 
      fzf
  ];
  
  environment.persistence."/persist" = { 
    hideMounts = true ; 
    directories = [
      "/etc/nixos"                       # NixOS
      "/etc/NetworkManager"              # NetworkManager
      { directory = "/var/lib/colord"; user = "colord"; group = "colord"; mode = "u=rwx,g=rx,o="; }        # Colord Profiles
      "/var/lib/bluetooth"               # Bluetooth
      "/var/lib/docker"                  # Docker
    ];
    files = [   
      "/etc/ssh/ssh_host_*"                 # SSH
      "/etc/machine-id"                     # Needed for SystemD Journal
      "/var/lib/NetworkManager/secret_key"  # Network Manager
      "/var/lib/NetworkManager/seen-bssids" # Network Manager
      "/var/lib/NetworkManager/timestamps"  # Network Manager
      { file = "/etc/nix/id_rsa"; parentDirectory = { mode = "u=rwx,g=,o="; }; } # Nix

    ];
  };
}

Setup Impermanance

   nixos-rebuild boot
   sudo mkdir -p /persist/var/lib/
   sudo cp -r {,/persist}/var/lib/docker
   nano /etc/nixos/configuration.nix 
   sudo mkdir -p /persist/etc/NetworkManager
   sudo cp -r {,/persist}/etc/NetworkManager/system-connections
   sudo mkdir -p /persist/var/lib/NetworkManager
   sudo cp /var/lib/NetworkManager/{secret_key,seen-bssids,timestamps} /persist/var/lib/NetworkManager/  
   sudo mkdir -p /persist/etc/ssh
   sudo cp -R /etc/ssh/ssh_host* /persist/etc/ssh/
   sudo cp -R {,/persist}/etc/nixos
   sudo cp -R {,/persist}/etc/NIXOS
   sudo cp {,/persist}/etc/machine-id
   reboot

Errors

Encrypted /boot

When mounting the ESP partition at /boot/efi instead of /boot grub picks the mounted pool0_0 filesystem and puts it in its configuration. Idea - Creating a seperate LUKS boot partition??