Skip to main content

Creating Opt-In BTRFS Installation with Snapshots

Inspired from this excellent blog post.

Variables

DISK_BOOT_SIZE_MB=512
DISK_SWAP_SIZE_GB=4
DISK=/dev/vda
HOSTNAME=beef

Wipe and Partition Disk

wipefs "${DISK}" -a -f
sgdisk --zap-all "${DISK}"
sgdisk --clear \
                --new=1:0:+"${DISK_BOOT_SIZE_MB}"MiB --typecode=1:ef00 --change-name=1:EFI \
                --new=2:0:+"${DISK_SWAP_SIZE_GB}"GiB --typecode=2:8200 --change-name=2:swap \
                --new=3:0:0 --typecode=3:8300 --change-name=3:pool0_0 \
                "${DISK}"

Create Encrypted Disk

cryptsetup --verify-passphrase -v luksFormat "${DISK}"3
cryptsetup open "${DISK}"3 pool0_0
mkswap "${DISK}"2
swapon "${DISK}"2
mkfs.btrfs -f /dev/mapper/pool0_0
mkfs.vfat "${DISK}"1

Create BTRFS Subvolumes

mount -t btrfs /dev/mapper/pool0_0 /mnt
btrfs subvolume create /mnt/root
mkdir -p /mnt/home
btrfs subvolume create /mnt/home/active
btrfs subvolume create /mnt/home/snapshots
btrfs subvolume create /mnt/nix
btrfs subvolume create /mnt/persist
mkdir -p /mnt/var_local
btrfs subvolume create /mnt/var_local/active
btrfs subvolume create /mnt/var_local/snapshots
btrfs subvolume create /mnt/var_log

Take a readonly snapshot of the root subvolume, which gets rolled back to on every boot.

btrfs subvolume snapshot -r /mnt/root /mnt/root-blank

Mount Subvolumes and Partitions

umount /mnt
mount -o subvol=root,compress=zstd,noatime /dev/mapper/pool0_0 /mnt

mkdir -p /mnt/home
mount -o subvol=home/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home

mkdir -p /mnt/nix
mount -o subvol=nix,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/nix

mkdir -p /mnt/persist
mount -o subvol=persist,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/persist

mkdir -p /mnt/var/local
mount -o subvol=var_local/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local

mkdir -p /mnt/var/log
mount -o subvol=var_log,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/log

mkdir -p /mnt/boot
mount -o defaults,nosuid,nodev,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro /dev/disk/by-partlabel/EFI /mnt/boot

Generate NixOS Hardware Configuration

nixos-generate-config --root /mnt

Force var_log subvolume to be available for boot

sed -i "/subvol=var_log/a\      neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix

Force BTRFS mount options

Nixos-generate config isn't smart enough to add options as per here.

sed -i "s|options = \[ \(.*\) \];|options = \[ \1 \"compress=zstd\" \"noatime\"  \];|g" /mnt/etc/nixos/hardware-configuration.nix

Create Base Configuration

/etc/nixos/configuration.nix

{ config, pkgs, ... }:

{
  imports =
    [ 
      ./hardware-configuration.nix
    ];

  boot = {
    loader = {
      efi = {
        canTouchEfiVariables = false;
      };
      grub = {
        enable = true;
        device = "nodev";  # No device for EFI
        efiSupport = true;
        useOSProber = false;
        efiInstallAsRemovable = true; # in case canTouchEfiVariables doesn't work for your system
      };
    };

    supportedFilesystems = [ 
      "btrfs"
      "fat" "vfat" "exfat" "ntfs" # Microsoft 
      "cifs"                      # Windows Network Share
    ];
  };

  networking = {
    hostName = "beef"; 
    networkmanager.enable = true;
  };

  i18n.defaultLocale = "en_US.UTF-8"; 

  services = {
    cinnamon.apps.enable = false ;
    openssh = {
      enable = true;
      settings = {
        PermitRootLogin = "yes" ;
      };
    };
    xserver = { 
      enable = true;
      layout = "us";
      libinput.enable = true;

      displayManager.gdm.enable = true ;
      desktopManager.cinnamon.enable = true;
    };
  };

  system.stateVersion = "23.05";

  time.timeZone = "America/Vancouver";

  users.users.dave = {
     isNormalUser = true;
     extraGroups = [ "wheel" ];
     packages = with pkgs; [
     ];
   };

}

Install NixOS

nixos-install

Booting into new system

Configure system and make any changes. Afterwords, run script below to detect changes in snapshot vs root since boot and start working on Persistence mapping.

Script

#!/usr/bin/env bash
# fs-diff.sh
mkdir -p /mnt
mount -o subvol=/ /dev/mapper/pool0_0 /mnt

set -euo pipefail

OLD_TRANSID=$(sudo btrfs subvolume find-new /mnt/root-blank 9999999)
OLD_TRANSID=${OLD_TRANSID#transid marker was }

sudo btrfs subvolume find-new "/mnt/root" "$OLD_TRANSID" |
sed '$d' |
cut -f17- -d' ' |
sort |
uniq |
while read path; do
  path="/$path"
  if [ -L "$path" ]; then
    : # The path is a symbolic link, so is probably handled by NixOS already
  elif [ -d "$path" ]; then
    : # The path is a directory, ignore
  else
    echo "$path"
  fi
done

Creating Persistence

Bluetooth

  systemd.tmpfiles.rules = [
    "L /var/lib/bluetooth - - - - /persist/var/lib/bluetooth"
  ];

##
sudo mkdir -p /persist/var/lib/bluetooth

Docker

  virtualisation = {
     docker.enable = true;
  };
  
    systemd.tmpfiles.rules = [
    "L /var/lib/docker - - - - /persist/var/lib/docker"
  ];

##
sudo mkdir -p /persist/var/lib/
sudo cp -r {,/persist}/var/lib/docker

Network Manager

  environment.etc."NetworkManager/system-connections" = {
    source = "/persist/etc/NetworkManager/system-connections/";
  };

  systemd.tmpfiles.rules = [
    "L /var/lib/NetworkManager/secret_key - - - - /persist/var/lib/NetworkManager/secret_key"
    "L /var/lib/NetworkManager/seen-bssids - - - - /persist/var/lib/NetworkManager/seen-bssids"
    "L /var/lib/NetworkManager/timestamps - - - - /persist/var/lib/NetworkManager/timestamps"
  ];
  
##
sudo mkdir -p /persist/etc/NetworkManager
sudo cp -r {,/persist}/etc/NetworkManager/system-connections
sudo mkdir -p /persist/var/lib/NetworkManager
sudo cp /var/lib/NetworkManager/{secret_key,seen-bssids,timestamps} /persist/var/lib/NetworkManager/

NixOS

 environment.etc = {
    nixos.source = "/persist/etc/nixos";
    NIXOS.source = "/persist/etc/NIXOS";
    machine-id.source = "/persist/etc/machine-id";
  };
    
sudo cp -R {,/persist}/etc/nixos
sudo cp -R {,/persist}/etc/NIXOS
sudo cp {,/persist}/etc/machine-id

OpenSSH

{
  services.openssh = {
    enable = true;
    hostKeys = [
      {
        path = "/persist/ssh/ssh_host_ed25519_key";
        type = "ed25519";
      }
      {
        path = "/persist/ssh/ssh_host_rsa_key";
        type = "rsa";
        bits = 4096;
      }
    ];
  };
}

Unmount commands

umount /mnt/boot
umount /mnt/home
umount /mnt/nix
umount /mnt/persist
umount /mnt/var/local
umount /mnt/var/log
umount /mnt

Restart Install over and over

umount /boot
umount /mnt/boot
umount /mnt/home
umount /mnt/nix
umount /mnt/persist
umount /mnt/var/local
umount /mnt/var/log
umount /mnt
mkfs.btrfs -f /dev/mapper/pool0_0
mkfs.vfat /dev/vda1
mount -t btrfs /dev/mapper/pool0_0 /mnt
btrfs subvolume create /mnt/root
mkdir -p /mnt/home
btrfs subvolume create /mnt/home/active
btrfs subvolume create /mnt/home/snapshots
btrfs subvolume create /mnt/nix
btrfs subvolume create /mnt/persist
mkdir -p /mnt/var_local
btrfs subvolume create /mnt/var_local/active
btrfs subvolume create /mnt/var_local/snapshots
btrfs subvolume create /mnt/var_log
btrfs subvolume snapshot -r /mnt/root /mnt/root-blank
umount /mnt
mount -o subvol=root,compress=zstd,noatime /dev/mapper/pool0_0 /mnt
mkdir -p /mnt/home
mount -o subvol=home/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home
mkdir -p /mnt/nix
mount -o subvol=nix,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/nix
mkdir -p /mnt/persist
mount -o subvol=persist,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/persist
mkdir -p /mnt/var/local
mount -o subvol=var_local/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local
mkdir -p /mnt/var/log
mount -o subvol=var_log,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/log
mkdir -p /mnt/boot
mount -o defaults,nosuid,nodev,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro /dev/disk/by-partlabel/EFI /mnt/boot
nixos-generate-config --root /mnt
sed -i "/subvol=var_log/a\      neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix 
sed -i "s|options = \[ \(.*\) \];|options = \[ \1 \"compress=zstd\" \"noatime\"  \];|g" /mnt/etc/nixos/hardware-configuration.nix 
rm -rf /mnt/etc/nixos/configuration.nix
cp -R /tmp/configuration.nix /mnt/etc/nixos/
nano /mnt/etc/nixos/configuration.nix
nixos-install
Back to top