From f8c3027812718db56b0cbdad137881f44bc1264a Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Sat, 21 Nov 2020 10:53:32 +1000 Subject: [PATCH 1/4] openstack-metadata-fetcher: stop lying in log message --- nixos/modules/virtualisation/openstack-metadata-fetcher.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/virtualisation/openstack-metadata-fetcher.nix b/nixos/modules/virtualisation/openstack-metadata-fetcher.nix index b531787c31a2..b8f78efb98ab 100644 --- a/nixos/modules/virtualisation/openstack-metadata-fetcher.nix +++ b/nixos/modules/virtualisation/openstack-metadata-fetcher.nix @@ -3,7 +3,7 @@ metaDir=${targetRoot}etc/ec2-metadata mkdir -m 0755 -p "$metaDir" - echo "getting EC2 instance metadata..." + echo "getting instance metadata..." if ! [ -e "$metaDir/ami-manifest-path" ]; then wget ${wgetExtraOptions} -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path From 8c39655de3f0ae5adbbfb3b9c5be412c4c3801d6 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Sat, 21 Nov 2020 11:40:38 +1000 Subject: [PATCH 2/4] {ec2,openstack}-metadata-fetcher: introduce wget_imds function --- .../modules/virtualisation/ec2-metadata-fetcher.nix | 12 ++++++++---- .../virtualisation/openstack-metadata-fetcher.nix | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/nixos/modules/virtualisation/ec2-metadata-fetcher.nix b/nixos/modules/virtualisation/ec2-metadata-fetcher.nix index 812e93ec4aab..0b110d375a41 100644 --- a/nixos/modules/virtualisation/ec2-metadata-fetcher.nix +++ b/nixos/modules/virtualisation/ec2-metadata-fetcher.nix @@ -61,19 +61,23 @@ echo "getting EC2 instance metadata..." + wget_imds() { + wget ${wgetExtraOptions} --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" "$@"; + } + if ! [ -e "$metaDir/ami-manifest-path" ]; then - wget ${wgetExtraOptions} --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path + wget_imds -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path fi if ! [ -e "$metaDir/user-data" ]; then - wget ${wgetExtraOptions} --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data" + wget_imds -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data" fi if ! [ -e "$metaDir/hostname" ]; then - wget ${wgetExtraOptions} --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname + wget_imds -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname fi if ! [ -e "$metaDir/public-keys-0-openssh-key" ]; then - wget ${wgetExtraOptions} --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key + wget_imds -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key fi '' diff --git a/nixos/modules/virtualisation/openstack-metadata-fetcher.nix b/nixos/modules/virtualisation/openstack-metadata-fetcher.nix index b8f78efb98ab..2aff8ccd2c26 100644 --- a/nixos/modules/virtualisation/openstack-metadata-fetcher.nix +++ b/nixos/modules/virtualisation/openstack-metadata-fetcher.nix @@ -5,19 +5,23 @@ echo "getting instance metadata..." + wget_imds() { + wget ${wgetExtraOptions} "$@" + } + if ! [ -e "$metaDir/ami-manifest-path" ]; then - wget ${wgetExtraOptions} -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path + wget_imds -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path fi if ! [ -e "$metaDir/user-data" ]; then - wget ${wgetExtraOptions} -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data" + wget_imds -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data" fi if ! [ -e "$metaDir/hostname" ]; then - wget ${wgetExtraOptions} -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname + wget_imds -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname fi if ! [ -e "$metaDir/public-keys-0-openssh-key" ]; then - wget ${wgetExtraOptions} -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key + wget_imds -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key fi '' From 43bfd7e5b1a70fe4be9b9c077eccb15fd50f6edc Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Sat, 21 Nov 2020 11:44:25 +1000 Subject: [PATCH 3/4] {ec2,openstack}-metadata-fetcher: unconditionally fetch metadata The metadata fetcher scripts run each time an instance starts, and it is not safe to assume that responses from the instance metadata service (IMDS) will be as they were on first boot. Example: an EC2 instance can have its user data changed while the instance is stopped. When the instance is restarted, we want to see the new user data applied. --- .../virtualisation/ec2-metadata-fetcher.nix | 24 +++++++------------ .../openstack-metadata-fetcher.nix | 24 +++++++------------ 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/nixos/modules/virtualisation/ec2-metadata-fetcher.nix b/nixos/modules/virtualisation/ec2-metadata-fetcher.nix index 0b110d375a41..dca5c2abd4e0 100644 --- a/nixos/modules/virtualisation/ec2-metadata-fetcher.nix +++ b/nixos/modules/virtualisation/ec2-metadata-fetcher.nix @@ -8,9 +8,14 @@ # Make sure that every package you depend on here is already listed as # a channel blocker for both the full-sized and small channels. # Otherwise, we risk breaking user deploys in released channels. +# +# Also note: OpenStack's metadata service for its instances aims to be +# compatible with the EC2 IMDS. Where possible, try to keep the set of +# fetched metadata in sync with ./openstack-metadata-fetcher.nix . '' metaDir=${targetRoot}etc/ec2-metadata mkdir -m 0755 -p "$metaDir" + rm -f "$metaDir/*" get_imds_token() { # retry-delay of 1 selected to give the system a second to get going, @@ -65,19 +70,8 @@ wget ${wgetExtraOptions} --header "X-aws-ec2-metadata-token: $IMDS_TOKEN" "$@"; } - if ! [ -e "$metaDir/ami-manifest-path" ]; then - wget_imds -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path - fi - - if ! [ -e "$metaDir/user-data" ]; then - wget_imds -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data" - fi - - if ! [ -e "$metaDir/hostname" ]; then - wget_imds -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname - fi - - if ! [ -e "$metaDir/public-keys-0-openssh-key" ]; then - wget_imds -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key - fi + wget_imds -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path + wget_imds -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data" + wget_imds -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname + wget_imds -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key '' diff --git a/nixos/modules/virtualisation/openstack-metadata-fetcher.nix b/nixos/modules/virtualisation/openstack-metadata-fetcher.nix index 2aff8ccd2c26..8c191397cf9a 100644 --- a/nixos/modules/virtualisation/openstack-metadata-fetcher.nix +++ b/nixos/modules/virtualisation/openstack-metadata-fetcher.nix @@ -1,7 +1,12 @@ { targetRoot, wgetExtraOptions }: + +# OpenStack's metadata service aims to be EC2-compatible. Where +# possible, try to keep the set of fetched metadata in sync with +# ./ec2-metadata-fetcher.nix . '' metaDir=${targetRoot}etc/ec2-metadata mkdir -m 0755 -p "$metaDir" + rm -f "$metaDir/*" echo "getting instance metadata..." @@ -9,19 +14,8 @@ wget ${wgetExtraOptions} "$@" } - if ! [ -e "$metaDir/ami-manifest-path" ]; then - wget_imds -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path - fi - - if ! [ -e "$metaDir/user-data" ]; then - wget_imds -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data" - fi - - if ! [ -e "$metaDir/hostname" ]; then - wget_imds -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname - fi - - if ! [ -e "$metaDir/public-keys-0-openssh-key" ]; then - wget_imds -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key - fi + wget_imds -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path + wget_imds -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data" + wget_imds -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname + wget_imds -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key '' From 6fd871dec4dba27611e6eade68f1d160af837dda Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Sat, 21 Nov 2020 11:59:30 +1000 Subject: [PATCH 4/4] rl-21.03: describe EC2 instance user/meta data reloading --- nixos/doc/manual/release-notes/rl-2103.xml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-2103.xml b/nixos/doc/manual/release-notes/rl-2103.xml index 55c1229a164d..5bdc44365ed5 100644 --- a/nixos/doc/manual/release-notes/rl-2103.xml +++ b/nixos/doc/manual/release-notes/rl-2103.xml @@ -211,6 +211,22 @@ and slaptest is buggy with schemas directly in the config file. + + + Amazon EC2 and OpenStack Compute (nova) images now re-fetch instance meta data and user data from the instance + metadata service (IMDS) on each boot. For example: stopping an EC2 instance, changing its user data, and + restarting the instance will now cause it to fetch and apply the new user data. + + + + Specifically, /etc/ec2-metadata is re-populated on each boot. Some NixOS scripts that read + from this directory are guarded to only run if the files they want to manipulate do not already exist, and so + will not re-apply their changes if the IMDS response changes. Examples: root's SSH key is + only added if /root/.ssh/authorized_keys does not exist, and SSH host keys are only set from + user data if they do not exist in /etc/ssh. + + +