Chris Martin
Monadic Party 
2018 June 14-15
Haskell is absolutely good for servers
NixOS takes all the pain out of provisioning servers
Not wizardry, a little boilerplate gets you started
General Linux familiarity
Concepts: What is a process, web server, SSH
Reading basic Haskell
We won’t use Linux containers / Docker
We use AWS, but this class is not about AWS
Not analyzing performance
Not using the Nix 2.0 nix command
PHP (2003–2006) dumb projects
https://use.foldapp.com
Ubuntu
Salthttps://typeclasses.com
NixOSThe Purely Functional Software Deployment Model, Eelco Dolstra, 2006
This thesis is about getting computer programs from one machine to another—and having them still work when they get there.
This should be a simple problem. For instance, if the software consists of a set of files, then deployment should be a simple matter of copying those to the target machines.
PATH?The Purely Functional Software Deployment Model, Eelco Dolstra, 2006
The main idea of the Nix approach is to store software components in isolation from each other in a central component store, under path names that contain cryptographic hashes of all inputs involved in building the component, such as
/nix/store/rwmfbhb2znwp...-firefox1.0.4[…] this prevents undeclared dependencies and enables support for side-by-side existence of component versions and variants.
The hash is computed over all inputs, including the following:
- The sources of the components.
- The script that performed the build.
- Any arguments or environment variables passed to the build script.
- All build time dependencies, typically including the compiler, linker, any libraries used at build time, standard Unix tools such as
cpandtar, the shell, and so on.
| Make | Nix | |
|---|---|---|
| Build unit | Targets | Derivations |
| Build file | List of targets | Lazy λ-calculus |
| Basis for rebuilds | Timestamps of build inputs | Hashes of build inputs |
| Where builds results go | Where you specify | In the Nix store (/nix/store/...) |
| How to remove old files | A target called clean |
nix-collect-garbage |
| Composing build files | include |
import |
| External dependencies | ? | nixpkgs |
https://archives.haskell.org/projects.haskell.org/diagrams/gallery/Hilbert.html
Makefileall: hilbert-curve.png
sourceUrl = https://archives.haskell.org/projects.haskell.org/diagrams/gallery/Hilbert.lhs
hilbert-curve.lhs:
wget $(sourceUrl) --output-document $@
hilbert-curve.svg: hilbert-curve.lhs
runhaskell --ghc-arg=-XTypeFamilies $< --output $@ --width 250
hilbert-curve.png: hilbert-curve.svg
convert $< $@
clean:
rm -f *.lhs *.svg *.png
.PHONY: cleandefault.nixlet
pkgs = import <nixpkgs> {};
src = pkgs.fetchurl {
url = https://archives.haskell.org/projects.haskell.org/diagrams/gallery/Hilbert.lhs;
sha256 = "0bplymiysf4k9vlw2kf5dfzgk1w2r0w5pbr8m24wjk045rml5hqq";
};
ghc = pkgs.haskellPackages.ghcWithPackages (p: [ p.diagrams ]);
svg = pkgs.runCommand "hilbert-curve.svg"
{ buildInputs = [ ghc ]; }
"runhaskell --ghc-arg=-XTypeFamilies ${src} -o $out -w 250";
png = pkgs.runCommand "hilbert-curve.png"
{ buildInputs = [ pkgs.imagemagick ]; }
"convert ${svg} $out";
in
pngMakefileMakefile doesn’t trigger a rebuildclean rule ourselves/etc/nixos/configuration.nix
https://nixos.org/nixos/download.html
WHAT 1GB of mem is good enough? 🤔 skepticalface is skeptical.
configuration.nix for EC2 instanceshttps://nixos.org/nixos/options.html
bootstrap.nix{
imports = [ <nixpkgs/nixos/modules/virtualisation/amazon-image.nix> ];
config = {
ec2.hvm = true;
# Run SSH on a weird port
services.openssh.ports = [ 46861 ];
networking.firewall.allowedTCPPorts = [ 46861 ];
# Don't allow anyone to SSH using a password
services.openssh.passwordAuthentication = false;
# Let admins sudo without entering a password
security.sudo.wheelNeedsPassword = false;
# Let admins upload unsigned Nix packages
nix.trustedUsers = [ "@wheel" ];
users.users.chris = {
isNormalUser = true;
extraGroups = [ "wheel" ];
openssh.authorizedKeys.keys = [
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDFJ4UYsEh+JZQGCMdbNrKfjH1F3rwKBRewwgaehwnijBADYSJ8iwDZji09vVfCxSQSMjZJS54mEBtjcOBOpM7+mR585wI6jhsfdsNqNwzJdxV47Bi4jAkg7XlWf9IYv7EUhRzsKGdoSqefh/7bN6MPcJQ9ccHKqBxtmGJ6eHfgLmgnb8+ozwDlwQKz5QDtdEnt8TUqucUB4AOyReBV7GRnwkTyGCForb5nhTftuVi7GO1qApJKBIpYlC1gbuCWDX9CIl7IzfAMyng6u6Ty9x31ZWKA0sJzRIX5cw3e8Ct7sWzZB3O/2FOwjyYadqTRQdR472Dz/f6mqqIl1ioxzfXRfh33bREg2opLc6bnYaWTXY6aAc5/wUbC7z4CTKBGZJHxY5mrRSlpQ2Rn8EvgyyxgxokLdTZqoiKw/tSmE9Mlle5JGh+m8agGe41dszZxBf41j/ORE+N5p0k02fvUWuG0PL3aFE77qUbOgxxXOYMtBV0YiJPzeBXDGrkW1wqKC2voJ6PuCZWOHaLxDqkUDgAMYyGMKoj5C53OZneVeSMgZG+/lxygAduyBx/RfQYrt4WsPfPnhl95Kxx8PTYuFfLXmcMNMhZ7rYW+Thvo40W+VjiqTUSCxLHr16SFSOj2mGl0A29VPPHA3H+ckprCo8pldPo3AYrwkV/zHlyLjuEQfQ== renzo-1"
];
};
users.users.julie = {
isNormalUser = true;
extraGroups = [ "wheel" ];
openssh.authorizedKeys.keys = [
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDUA5QiRaMriKQrvA1H74UsJcnGieAcHYrHp27KBZjOhhKvinTkNyErY5JNFbgrCh3oC+6HeMSUOp+5qb2/hF0aO6kdYbGnYB3BchrbsPYFDccqW9eQONXdvYsa4mavfLzHIvXktkxVsd71CBRXqVFxxolYps99iJvFfhmgnn9Iz+zUIuOSRCPQ/Bcxbh3+fZHD2wlPpOZ352V3/utR6n6nlhK5W1Gq/mp5dmE32zCEOEse85xXQNxJpjdhIDZbu8PnNo2LuYIfRBFzIj/Gh6MIHJU3gQskt8MvKA+POdseflPLznxoDWYT3FqBO+agObr3FGnlnwpi1g9ym+U8T6SNFGUwR7cX5VkegLNSl7FTLMtaXoqUTKhBQH+ZIpNwlyepYnFo1BHR3IgsFDSaD/zLUjesBQIw6j+mtDCK9P3EnUcXo5v04OuGoyqTtAF4TTAz2kuC8ZsCs0cQEZRIoXqVIRcvPmlFr208o0SeGakCHVxIj6VnrQ+aHisVwuGAkq+Kb5mwE6b0xvuFGHo+bqHIUePymWQihbh0pZpYeSaJTLb2YG17HmE2rUFeO66CADmIMi2EWEI0xisT/e9FoqDcMnG/Z5sLBBDkerXwyfLA63zAkF0P3LtWX1Q0vYFhCn+VtMwE56qW/Z0sDdNiVwxn/a1GGh12gqXkNHBEePYjoQ== doriangray"
];
};
};
}
❮bash❯ aws ec2 run-instances [... a bunch of args ...]NixOS is a Nix package
Build NixOS locally, upload to the server
server.nix❮bash❯ nix-build server.nix
[... a bunch of build output ...]
/nix/store/bbcbh0vv64v5bgdhdd4a8fw3p438iiam-nixos-system-unnamed-18.03.132618.0f73fef53a9
❮bash❯ ls -lh
ls -lh
total 12K
-rw-r--r-- 1 chris users 2.2K Jun 7 02:30 bootstrap.nix
lrwxrwxrwx 1 chris users 89 Jun 7 02:35 result -> /nix/store/pvf3l9x8m2fnw7l07g1yxw1iilj3m2rp-nixos-system-unnamed-18.03.132618.0f73fef53a9
-rw-r--r-- 1 chris users 176 Jun 7 02:30 server.nix❮bash❯ ls -lh result/
total 60K
dr-xr-xr-x 2 root root 4.0K Dec 31 1969 bin
dr-xr-xr-x 2 root root 4.0K Dec 31 1969 fine-tune
-r-xr-xr-x 1 root root 13K Dec 31 1969 activate
lrwxrwxrwx 1 root root 91 Dec 31 1969 append-initrd-secrets -> /nix/store/m2jf41zpw52i10j01p4wij6j2h22721j-append-initrd-secrets/bin/append-initrd-secrets
-r--r--r-- 1 root root 0 Dec 31 1969 configuration-name
lrwxrwxrwx 1 root root 51 Dec 31 1969 etc -> /nix/store/iwpgvdg2i4vv6a0vd20d70425s0rifdh-etc/etc
-r--r--r-- 1 root root 0 Dec 31 1969 extra-dependencies
lrwxrwxrwx 1 root root 65 Dec 31 1969 firmware -> /nix/store/grzr4r1ajmnjjnq049r719jbj2p8zbcp-firmware/lib/firmware
-r-xr-xr-x 1 root root 4.9K Dec 31 1969 init
-r--r--r-- 1 root root 9 Dec 31 1969 init-interface-version
lrwxrwxrwx 1 root root 57 Dec 31 1969 initrd -> /nix/store/bnqw7sxz3pnakni5hmj22dqrqvjbbq1m-initrd/initrd
lrwxrwxrwx 1 root root 65 Dec 31 1969 kernel -> /nix/store/y0jcw44kw1962bdfnsayz1r6jsgz2sqj-linux-4.14.44/bzImage
lrwxrwxrwx 1 root root 58 Dec 31 1969 kernel-modules -> /nix/store/yw9zkbwkb76ypb0gvvakg0lkv1zirqg7-kernel-modules
-r--r--r-- 1 root root 51 Dec 31 1969 kernel-params
-r--r--r-- 1 root root 24 Dec 31 1969 nixos-version
lrwxrwxrwx 1 root root 55 Dec 31 1969 sw -> /nix/store/j85qcbjfwmis0c75qwbvxhirlj02kjmw-system-path
-r--r--r-- 1 root root 12 Dec 31 1969 system
lrwxrwxrwx 1 root root 55 Dec 31 1969 systemd -> /nix/store/cya0k4g78bd518wzgqghkk6srlvl2jgv-systemd-237deploy.hsmainmain :: IO ()
main =
sh $ do
server <- getServer -- Read the server address from a file
path <- build -- (1) Build NixOS for our server
upload server path -- (2) Upload the build to the server
activate server path -- (3) Start running the new version
getServerbuilduploadactivate{-# LANGUAGE QuasiQuotes #-}
import NeatInterpolation (text)
-- Start running the new version of NixOS on the server.
activate :: Server -> NixOS -> Shell ()
activate (Server server) (NixOS path) =
procs command args empty
where
command = "ssh"
args = [server, remoteCommand]
profile = "/nix/var/nix/profiles/system"
remoteCommand = [text|
nix-env --profile $profile --set $path
$profile/bin/switch-to-configuration switch
|]❮bash❯ ./deploy.hs
[... a bunch of build output ...]
updating GRUB 2 menu...
activating the configuration...
setting up /etc...
setting up tmpfilesdefault.nixshell.nixmonadic-party
├── default.nix
├── ghc-version.nix
├── monadic-party.cabal
├── shell.nix
├── app
│ └── party-server.hs
└── lib
└── PartyServer.hsmonadic-party/ghc-version.nixmonadic-party/default.nix{ pkgs }:
let
ghc = pkgs.haskell.packages.${import ./ghc-version.nix};
developPackage =
{ root
, source-overrides ? {}
, overrides ? self: super: {}
}:
let
extensions = pkgs.lib.composeExtensions
(ghc.packageSourceOverrides source-overrides)
overrides;
ghc' = ghc.extend extensions;
package = ghc'.callCabal2nix (builtins.baseNameOf root) root {};
in
pkgs.haskell.lib.overrideCabal package (drv: {
enableSharedExecutables = false;
isLibrary = false;
doHaddock = false;
postFixup = "rm -rf $out/lib $out/nix-support $out/share/doc";
});
in
developPackage {
root = ./.;
source-overrides = {
# Any specific Haskell package versions you need to pin go here, e.g.:
# scotty = "0.11.1";
};
}monadic-party/shell.nix (basic)monadic-party/shell.nix (with dev tools)let
pkgs = import <nixpkgs> { config.allowUnfree = true; };
monadic-party = import ./default.nix { inherit pkgs; };
ghc = pkgs.haskell.packages.${import ./ghc-version.nix};
cabal = ghc.cabal-install;
party-ghci = pkgs.writeScriptBin "party-ghci"
''
${cabal}/bin/cabal new-repl "$@"
'';
party-ghcid = pkgs.writeScriptBin "party-ghcid"
''
${ghc.ghcid}/bin/ghcid --command "${cabal}/bin/cabal new-repl" "$@"
'';
devTools = [
party-ghci
party-ghcid
];
in
monadic-party.env.overrideAttrs (attrs: {
buildInputs = attrs.buildInputs ++ devTools;
})❮bash❯ nix-shell --pure --run 'party-ghci'
In order, the following will be built (use -v for more details):
- monadic-party-0 (lib) (first run)
Preprocessing library for monadic-party-0..
GHCi, version 8.2.2: http://www.haskell.org/ghc/ :? for help
Loaded GHCi configuration from /home/chris/.ghc/ghci.conf
[1 of 1] Compiling PartyServer ( lib/PartyServer.hs, interpreted )
Ok, one module loaded.
λ>monadic-party/monadic-party.cabalname: monadic-party
version: 0
cabal-version: >= 1.10
build-type: Simple
library
hs-source-dirs: lib
exposed-modules:
MonadicParty.Scotty
build-depends:
base
, neat-interpolation
, scotty
, text
ghc-options: -Wall
default-language: Haskell2010
executable party-scotty
hs-source-dirs: app
main-is: party-scotty.hs
build-depends:
base
, monadic-party
ghc-options: -Wall
default-language: Haskell2010monadic-party/lib/MonadicParty/Scotty.hs{-# LANGUAGE OverloadedStrings, QuasiQuotes #-}
module MonadicParty.Scotty (main) where
-- neat-interpolation
import NeatInterpolation
-- scotty
import qualified Web.Scotty as Scotty
-- text
import qualified Data.Text.Lazy
main :: IO ()
main =
Scotty.scotty 8000 scottyApp
scottyApp :: Scotty.ScottyM ()
scottyApp =
Scotty.get "/scotty" $
Scotty.html (Data.Text.Lazy.fromStrict [text|
<!doctype html>
<html>
<head></head>
<body>
Hello, I am a very basic Scotty demo,
serving (internally) on port 8000.
</body>
</html>
|])server.nixmodules/party-scotty.nix{ pkgs, ... }:
{
imports = [
];
config = {
systemd.services.party-scotty = {
enable = true;
description = "Monadic Party - Scotty";
wantedBy = ["multi-user.target"];
environment = {
LC_ALL = "en_US.UTF-8";
LOCALE_ARCHIVE = "${pkgs.glibcLocales}/lib/locale/locale-archive";
};
serviceConfig = {
User = "monadic-party";
Restart = "on-failure";
ExecStart = "${import ../monadic-party { inherit pkgs; }}/bin/party-scotty";
};
};
};
}modules/nginx.nix{ pkgs, ... }:
{
imports = [
];
config = {
networking.firewall.allowedTCPPorts = [ 80 443 ];
services.nginx = {
enable = true;
recommendedGzipSettings = true;
recommendedOptimisation = true;
recommendedProxySettings = true;
recommendedTlsSettings = true;
virtualHosts."monadic-party.chris-martin.org" = {
enableACME = true;
addSSL = true;
locations."/scotty".proxyPass =
"http://localhost:8000";
};
};
};
}./deploy.hs that we walked through yesterdayHaskell project setup:
monadic-party.cabaldefault.nix specifies how to build a package for deploymentshell.nix defines our development environmentNix tools we’ve seen:
nix-shellnix-buildnix-copy-closurenix-collect-garbageNixOS modules on our server:
modules/bootstrap.nixmodules/party-scotty.nixmodules/nginx.nix502 Bad Gateway
modules/party-socket.nix{ pkgs, ... }:
{
config = {
systemd.services.party-socket = {
enable = true;
description = "Monadic Party - Socket";
wantedBy = ["multi-user.target"];
requires = ["party-socket.socket"];
environment = {
LC_ALL = "en_US.UTF-8";
LOCALE_ARCHIVE = "${pkgs.glibcLocales}/lib/locale/locale-archive";
};
serviceConfig = {
User = "monadic-party";
Restart = "on-failure";
ExecStart = "${import ../monadic-party { inherit pkgs; }}/bin/party-socket";
};
};
systemd.sockets.party-socket = {
wantedBy = ["sockets.target"];
socketConfig = {
ListenStream = "/run/party-socket.socket";
};
};
};
}socket-activation Haskell librarysocket-activation Haskell library (2)-- | Return a list of activated sockets, if the program was started
-- with socket activation. The sockets are in the same order as in
-- the associated .socket file.
getActivatedSockets :: IO (Maybe [Socket])
getActivatedSockets =
runMaybeT $ do
listenPid <- read <$> MaybeT (getEnv "LISTEN_PID")
listenFDs <- read <$> MaybeT (getEnv "LISTEN_FDS")
myPid <- lift getProcessID
guard $ listenPid == myPid
mapM makeSocket [fdStart .. fdStart + listenFDs - 1]
where
fdStart = 3-- | Run a scotty application using the warp server.
scotty :: Port -> ScottyM () -> IO ()
-- | Run a scotty application using the warp server, passing extra options.
scottyOpts :: Options -> ScottyM () -> IO ()
-- | Run a scotty application using the warp server, passing extra options,
-- and listening on the provided socket.
scottySocket :: Options -> Socket -> ScottyM () -> IO ()Options
data-default-class Haskell libraryTypeApplications{-# LANGUAGE TypeApplications #-}def @Web.Scotty.Optionsnetwork Haskell librarymonadic-party/lib/MonadicParty/Socket.hs{-# LANGUAGE OverloadedStrings, QuasiQuotes,
ScopedTypeVariables, TypeApplications #-}
module MonadicParty.Socket (main) where
-- base
import System.Exit (die)
-- data-default-class
import Data.Default.Class (def)
-- neat-interpolation
import NeatInterpolation
-- network
import Network.Socket (Socket)
import qualified Network.Socket as Socket
-- scotty
import qualified Web.Scotty as Scotty
-- socket-activation
import Network.Socket.Activation (getActivatedSockets)
-- text
import qualified Data.Text.Lazy
main :: IO ()
main = do
socketsMaybe :: Maybe [Socket] <- getActivatedSockets
case socketsMaybe of
Just [socket] ->
do
Socket.setNonBlockIfNeeded (Socket.fdSocket socket)
Scotty.scottySocket (def @Scotty.Options) socket scottyApp
Just sockets -> die ("Wrong number of sockets: " ++
show (length sockets))
Nothing -> die "Not socket activated"
scottyApp :: Scotty.ScottyM ()
scottyApp =
Scotty.get "/socket" $
Scotty.html (Data.Text.Lazy.fromStrict [text|
<!doctype html>
<html>
<head></head>
<body>
This one is socket activated.
</body>
</html>
|])main :: IO ()
main = do
socketsMaybe :: Maybe [Socket] <- getActivatedSockets
case socketsMaybe of
Nothing ->
Scotty.scotty 8001 scottyApp
Just [socket] ->
do
Socket.setNonBlockIfNeeded (Socket.fdSocket socket)
Scotty.scottySocket (def @Scotty.Options) socket scottyApp
Just sockets -> die ("Wrong number of sockets: " ++
show (length sockets))server.nixlet
nixos = import <nixpkgs/nixos> {
configuration = {
imports = [
./modules/bootstrap.nix
./modules/party-scotty.nix
./modules/party-socket.nix
./modules/nginx.nix
];
config = {
networking.hostName = "monadic-party";
nixpkgs.config.allowUnfree = true;
users.users.monadic-party = {};
};
};
};
in
nixos.systemmodules/nginx.nix{ pkgs, ... }:
{
imports = [
];
config = {
networking.firewall.allowedTCPPorts = [ 80 443 ];
services.nginx = {
enable = true;
recommendedGzipSettings = true;
recommendedOptimisation = true;
recommendedProxySettings = true;
recommendedTlsSettings = true;
virtualHosts."monadic-party.chris-martin.org" = {
enableACME = true;
addSSL = true;
locations."/scotty".proxyPass =
"http://localhost:8000";
locations."/socket".proxyPass =
"http://unix:/run/party-socket.socket";
};
};
};
}❮bash❯ curl --unix-socket /run/party-socket.socket dummydomain/socket
helloNIX_PATHTypically NIX_PATH=$HOME/.nix-defexpr/channels
<nixpkgs> = $NIX_PATH/nixpkgs
<nixpkgs/nixos> = $NIX_PATH/nixpkgs/nixos
nix-channel --update changes the nixpkgs expression!
NIX_PATH (2)NIX_PATH, more precisely, is a colon-separated list where each entry is either
name=path where path is a Nix channelnixpkgs.nix# nix-prefetch-url --unpack https://github.com/NixOS/nixpkgs/archive/<rev>.tar.gz
(import <nixpkgs> { }).fetchFromGitHub {
owner = "NixOS";
repo = "nixpkgs";
# NixOS 18.03, 2018 Mar 18
rev = "0e7c9b32817e5cbe61212d47a6cf9bcd71789322";
sha256 = "1dm777cmlhqcwlrq8zl9q2d87h3p70rclpvq36y43kp378f3pd0y";
}deploy.hsmain :: IO ()
main =
sh $ do
nixpkgs <- getNixpkgs -- Get the pinned nixpkgs
setNixPath nixpkgs -- Set the NIX_PATH environment variable
server <- getServer -- Read the server address from a file
path <- build -- (1) Build NixOS for our server
upload server path -- (2) Upload the build to the server
activate server path -- (3) Start running the new versionNixpkgsgetNixpkgssetNixPathjournalctl -fjournalctl -u party-countnewLog :: IO LogHandlewriteToLog :: LogHandle -> Text -> IO ()runLogger :: LogHandle -> IO a
TextwriteToLogrunLoggerasync Haskell library