diff options
| author | Jasper Ras <jras@hostnet.nl> | 2025-04-04 14:31:53 +0200 |
|---|---|---|
| committer | Jasper Ras <jras@hostnet.nl> | 2025-04-04 14:31:53 +0200 |
| commit | f9034731fa234a4b5efa1d2d6147fe1e798b6d36 (patch) | |
| tree | 3e0ca7f2f73dbdafd666ce0d0a7b04d457394401 | |
| parent | 3f69286ef4b15161febb4a4bae085fb4c2c83bbf (diff) | |
vault backup: 2025-04-04 14:31:53
20 files changed, 306 insertions, 41 deletions
diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index d6443a3..f667601 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -4,67 +4,67 @@ "type": "split", "children": [ { - "id": "d7ac73dacb8efdbe", + "id": "b5921b94f3580090", "type": "tabs", "children": [ { - "id": "6bdd9a8324d3b0f0", + "id": "502f77e94f5ff48c", "type": "leaf", "state": { "type": "markdown", "state": { - "file": "Build custom modules as profiles to manage various NixOS hosts.md", + "file": "Managing Ceph on Devstack & OpenStack.md", "mode": "source", "backlinks": true, "source": false }, "icon": "lucide-file", - "title": "Build custom modules as profiles to manage various NixOS hosts" + "title": "Managing Ceph on Devstack & OpenStack" } }, { - "id": "84e32b3623ed3a00", + "id": "c10897c003c6bee7", "type": "leaf", "state": { "type": "markdown", "state": { - "file": "NixOS - Advantage of imperative container management.md", + "file": "OVN upgrade.md", "mode": "source", "backlinks": true, "source": false }, "icon": "lucide-file", - "title": "NixOS - Advantage of imperative container management" + "title": "OVN upgrade" } }, { - "id": "be0892c76b1d189a", + "id": "62d186e3d63b15e0", "type": "leaf", "state": { "type": "markdown", "state": { - "file": "Nix Inherit.md", + "file": "I'm worried that Freenet will completely go DOWN due to OVN upgrade.md", "mode": "source", "backlinks": true, "source": false }, "icon": "lucide-file", - "title": "Nix Inherit" + "title": "I'm worried that Freenet will completely go DOWN due to OVN upgrade" } }, { - "id": "1fb3c7b7af372a3c", + "id": "96a8cc05fb9592a6", "type": "leaf", "state": { "type": "markdown", "state": { - "file": "Distinction between imperative and declarative programming.md", + "file": "Network outage on single hypervisor causes Freenet to shutdown EVERYTHING.md", "mode": "source", "backlinks": true, "source": false }, "icon": "lucide-file", - "title": "Distinction between imperative and declarative programming" + "title": "Network outage on single hypervisor causes Freenet to shutdown EVERYTHING" } } ], @@ -99,7 +99,7 @@ "state": { "type": "search", "state": { - "query": "", + "query": "OVN upgrade", "matchingCase": false, "explainSearch": false, "collapseAll": false, @@ -219,35 +219,35 @@ "obsidian-excalidraw-plugin:New drawing": false } }, - "active": "1fb3c7b7af372a3c", + "active": "96a8cc05fb9592a6", "lastOpenFiles": [ - "Nix Inherit.md", - "Distinction between imperative and declarative programming.md", - "NixOS - Advantage of imperative container management.md", - "NixOS - Imperative container management.md", + "Making notes useful.md", + "Managing Ceph on Devstack & OpenStack.md", + "Network outage on single hypervisor causes Freenet to shutdown EVERYTHING.md", + "I'm worried that Freenet will completely go DOWN due to OVN upgrade.md", + "Random notes on Freenet deployment.md", + "OVN upgrade.md", + "Installing Ceph on devstack.md", + "Overview of Ceph.md", + "Gobs Ceph Client Key Access.md", + "Installing a devstack plugin.md", + "Rewriting subselect into just an additional join.md", + "Updating the image os admin user to fix password resets.md", "Freenet CPU steal graph.md", - "Test.md", + "Using Libvirt to manage QEMU image bitmaps.md", "Build custom modules as profiles to manage various NixOS hosts.md", - "EF02 boot partition.md", - "EF00 partition.md", - "Automatic configuration deployment with NixOS.md", - "Booting a NixOS on OpenStack.md", - "NixOS.md", - "Acronis Managed Backup provisioning flow.md", - "ESP", - "TODO's in Zettelkasten?.md", + "Nix Inherit.md", "Becoming a Senior.md", "A reason for becoming senior is that when Rutger is off I take most of his responsibilities.md", - "Overview of Ceph.md", - "Excalidraw/Intro to Ceph.excalidraw.md", - "Ceph on devstack.md", - "Excalidraw", - "Ideas versus facts.md", - "Installing a devstack plugin.md", - "Making notes useful.md", + "Acronis Managed Backup provisioning flow.md", + "Ceph.md", + "NixOS - Imperative container management.md", "NixOS - Declarative container management.md", "NixOS - Containers.md", "NixOS - Container networking.md", - "Practice summarizing and paraphrasing.md" + "EF02 partition.md", + "EF00 partition.md", + "ESP", + "Excalidraw" ] }
\ No newline at end of file diff --git a/Acronis Managed Backup provisioning flow.md b/.trash/Acronis Managed Backup provisioning flow.md index 7f2b4f6..7f2b4f6 100644 --- a/Acronis Managed Backup provisioning flow.md +++ b/.trash/Acronis Managed Backup provisioning flow.md diff --git a/.trash/Ceph.md b/.trash/Ceph.md new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/.trash/Ceph.md diff --git a/A reason for becoming senior is that when Rutger is off I take most of his responsibilities.md b/A reason for becoming senior is that when Rutger is off I take most of his responsibilities.md index 442b817..d76b919 100644 --- a/A reason for becoming senior is that when Rutger is off I take most of his responsibilities.md +++ b/A reason for becoming senior is that when Rutger is off I take most of his responsibilities.md @@ -3,5 +3,4 @@ tags: - work - role --- - Whenever Rutger has vacation most people he interacts with go to me. That feels a bit weird being the "medior" in our team that also has a "senior".
\ No newline at end of file diff --git a/Becoming a Senior.md b/Becoming a Senior.md index 5993f9e..64ab969 100644 --- a/Becoming a Senior.md +++ b/Becoming a Senior.md @@ -4,3 +4,5 @@ tags: - role --- This year I indicated that I would like to become senior. At the time Rutger gave a few reasons, mainly that my architectural skills are still lacking due to me going my own way during the development of gobs, making some (to Rutger) less than ideal decisions about API design and such. + +[[A reason for becoming senior is that when Rutger is off I take most of his responsibilities]] diff --git a/Booting a NixOS on OpenStack.md b/Booting a NixOS on OpenStack.md index 280b05b..d68a049 100644 --- a/Booting a NixOS on OpenStack.md +++ b/Booting a NixOS on OpenStack.md @@ -3,4 +3,4 @@ tags: - nixos - openstack --- -Booting a [[NixOS]] VM on openstack required me to partition a [[EF02 boot partition]] as well as an [[EF00 partition]] and install GRUB as a bootloader instead of systemd-boot.
\ No newline at end of file +Booting a [[NixOS]] VM on openstack required me to partition a [[EF02 partition]] as well as an [[EF00 partition]] and install GRUB as a bootloader instead of systemd-boot.
\ No newline at end of file diff --git a/EF00 partition.md b/EF00 partition.md index ccd93e1..0511d20 100644 --- a/EF00 partition.md +++ b/EF00 partition.md @@ -4,4 +4,6 @@ tags: - boot - efi --- -This partition is the EFI system partition a.k.a ESP. UEFI firmware loads the bootloader from here. This partition as opposite to the [[EF02 boot partition]] contains all tools needed to boot, such as the boot loader and kernel images.
\ No newline at end of file +This partition is the EFI system partition a.k.a ESP. UEFI firmware loads the bootloader from here. This partition as opposite to the [[EF02 partition]] contains all tools needed to boot, such as the boot loader and kernel images. + +Typically this partition is mounted as `/boot`.
\ No newline at end of file diff --git a/EF02 boot partition.md b/EF02 partition.md index e3b1197..e3b1197 100644 --- a/EF02 boot partition.md +++ b/EF02 partition.md diff --git a/Gobs Ceph Client Key Access.md b/Gobs Ceph Client Key Access.md new file mode 100644 index 0000000..d8f86c7 --- /dev/null +++ b/Gobs Ceph Client Key Access.md @@ -0,0 +1,13 @@ +--- +tags: + - work + - gobs + - ceph + - openstack +--- +[[Overview of Ceph]] + +Ceph credentials are typically stored in a keyring file under `/etc/ceph`. Nova has a a key there owned by nova/nova. +Instead of adding a new key I've added the `goba` user to the `nova` group and modified our hieradata to specify a mode which allows the group to read the keyring. + +https://gitlab.group.one/groupvps/hieradata/-/merge_requests/499/diffs diff --git a/I'm worried that Freenet will completely go DOWN due to OVN upgrade.md b/I'm worried that Freenet will completely go DOWN due to OVN upgrade.md new file mode 100644 index 0000000..fdf4aec --- /dev/null +++ b/I'm worried that Freenet will completely go DOWN due to OVN upgrade.md @@ -0,0 +1,8 @@ +--- +tags: + - work + - freenet +--- +[[OVN upgrade]] +[[Random notes on Freenet deployment]] +[[Freenet CPU steal graph]]
\ No newline at end of file diff --git a/Ceph on devstack.md b/Installing Ceph on devstack.md index 5dcbc62..c3d4a83 100644 --- a/Ceph on devstack.md +++ b/Installing Ceph on devstack.md @@ -1,11 +1,15 @@ --- tags: - - work - devstack - ceph + - openstack --- [[Overview of Ceph]] https://github.com/openstack/devstack-plugin-ceph Added `enable_plugin devstack-plugin-ceph https://opendev.org/openstack/devstack-plugin-ceph` to local.conf. + +Make sure we have [[Gobs Ceph Client Key Access]]. + +`sudo apt-get install python3-rados` on the docker image for goba.
\ No newline at end of file diff --git a/Installing a devstack plugin.md b/Installing a devstack plugin.md index 1220798..afeb431 100644 --- a/Installing a devstack plugin.md +++ b/Installing a devstack plugin.md @@ -5,4 +5,4 @@ tags: --- To install a devstack plugin add the corresponding `enable_plugin <name> <giturl> [gitref]` under the `[[local|localrc]]` section. -This way I successfully installed [[Ceph on devstack]] +This way I successfully installed [[Installing Ceph on devstack]] diff --git a/Managing Ceph on Devstack & OpenStack.md b/Managing Ceph on Devstack & OpenStack.md new file mode 100644 index 0000000..cad83e5 --- /dev/null +++ b/Managing Ceph on Devstack & OpenStack.md @@ -0,0 +1,42 @@ +--- +tags: + - ceph + - devstack + - openstack +--- +[[Installing Ceph on devstack]] + +The pool names are: +- volumes for cinder volumes +- images for glance images +- vms for nova "local storage" rbd volumes +- backups for cinder volume backups + +https://docs.ceph.com/en/reef/rbd/rados-rbd-cmds/ + +``` +ubuntu@gobs-devstack:~$ sudo rbd ls volumes +volume-5b6cc251-8f3e-4572-a8b9-52efa390ebc3 +volume-6f5e81dd-40f3-42f9-aad3-afcf5696387c +volume-ac39d5be-9606-4711-948e-e76c035e2a25 +``` + +Specify the pool in the `info` command: +``` +ubuntu@gobs-devstack:~$ sudo rbd info volumes/volume-6f5e81dd-40f3-42f9-aad3-afcf5696387c +rbd image 'volume-6f5e81dd-40f3-42f9-aad3-afcf5696387c': + size 10 GiB in 2560 objects + order 22 (4 MiB objects) + snapshot_count: 0 + id: 17d385edc7a94 + block_name_prefix: rbd_data.17d385edc7a94 + format: 2 + features: layering, exclusive-lock, object-map, fast-diff + op_features: + flags: + create_timestamp: Mon Mar 31 12:56:46 2025 + access_timestamp: Mon Mar 31 12:56:46 2025 + modify_timestamp: Mon Mar 31 12:56:46 2025 + parent: volumes/volume-5b6cc251-8f3e-4572-a8b9-52efa390ebc3@snapshot-3d1f148a-5fe4-4df2-87c3-4272565b82c2 + overlap: 10 GiB +```
\ No newline at end of file diff --git a/Network outage on single hypervisor causes Freenet to shutdown EVERYTHING.md b/Network outage on single hypervisor causes Freenet to shutdown EVERYTHING.md new file mode 100644 index 0000000..d342ab9 --- /dev/null +++ b/Network outage on single hypervisor causes Freenet to shutdown EVERYTHING.md @@ -0,0 +1,12 @@ +--- +tags: + - work + - freenet +--- +[[Freenet CPU steal graph]] +[[Random notes on Freenet deployment]] +[[I'm worried that Freenet will completely go DOWN due to OVN upgrade]] + +Recently we had added some hypervisors but one of them had invalid MTU set on the bond. Due to... me testing there, so rutger didn't reboot it. +During maintenance we started migrating workload onto it and their network started failing. +As a response to this Freenet ***automatically*** hit the **PANIC** switch and systemctl stopped about everything in ALL AVAILABILITY ZONES.
\ No newline at end of file diff --git a/OVN upgrade.md b/OVN upgrade.md new file mode 100644 index 0000000..8874ef4 --- /dev/null +++ b/OVN upgrade.md @@ -0,0 +1,40 @@ +--- +tags: + - work + - ovn + - upgrade +--- +https://docs.ovn.org/en/latest/intro/install/ovn-upgrades.html + +**Fail-safe upgrade procedure** +The ansible playbook does **NOT** encode the fail-safe procedure in such a way that it prevents us from skipping versions. It is **mandatory** that we ourselves correctly decide the version to upgrade to. + +**Is the issue resolved with group.one- prefixes on unit files and /etc/default?** +https://gitlab.group.one/groupvps/ovn-builder/-/merge_requests/11/diffs + +It seems to have been fixed using overrides in debian/rules that move stuff to the expected places after building. + +*After testing it looks like this is all done correctly now* + + +**Ansible playbook** +https://gitlab.group.one/groupvps/ansible/-/merge_requests/56/diffs + +- Makes a backup of OVN db's on networking nodes + - Creates files on the local system under /tmp + - Creates files on the remote system under /var/.ovn-backups +- Pins the controller version on both compute and networking nodes + - `ovs-vsctl set open_vswitch . external_ids:ovn-match-northd-version=true` +- Upgrades ovn-central, ovn-common and ovn-host on network nodes + - Installs packages + - restarts ovn-controller, ovn-northd, ovn-ovsdb-server-sb/nb + - Checks ovsdb-server is running + - Checks the socket file for existence +- Upgrades ovn controller on compute nodes + - Installs package + - restart ovn-controller + +**Invocation** +``` +ansible-playbook playbooks/failsafe_install_ovn.yml --limit vps_testpod --extra-vars "ovn_search_version=22.03.8+20250403.1251.35813e0b.systems.jammy1" +```
\ No newline at end of file diff --git a/Overview of Ceph.md b/Overview of Ceph.md index 0826f82..5a5505f 100644 --- a/Overview of Ceph.md +++ b/Overview of Ceph.md @@ -1,3 +1,8 @@ +--- +tags: + - ceph + - storage +--- Ceph provides Block Storage, Object Storage as well as shared filesystem. ![[Intro to Ceph.excalidraw]] diff --git a/Random notes on Freenet deployment.md b/Random notes on Freenet deployment.md new file mode 100644 index 0000000..e028053 --- /dev/null +++ b/Random notes on Freenet deployment.md @@ -0,0 +1,8 @@ +--- +tags: + - work + - freenet +--- +Uses [[Overview of Ceph]] as a volume backing store. Mehmet Top is the primary contact for this Ceph deployment. + +Their mbox VM's also connect to Quobyte storage.
\ No newline at end of file diff --git a/Rewriting subselect into just an additional join.md b/Rewriting subselect into just an additional join.md new file mode 100644 index 0000000..c82d27e --- /dev/null +++ b/Rewriting subselect into just an additional join.md @@ -0,0 +1,32 @@ +--- +tags: + - mysql +--- +Optimizing a query with subselects into a "flat" query with just additional joins. This query finds rows that are missing for instances. + +``` +select * +from instance_system_metadata ism1 +where ism1.instance_uuid in ( + select i.uuid + from instances i + left join instance_system_metadata ism + on ism.instance_uuid = i.uuid and ism.key = "image_os_admin_user" + where i.deleted_at is null and ism.id is null +) +and ism1.key = "image_hw_qemu_guest_agent"; +``` + +Can be rewritten more nicely with an additional join +``` +select i.uuid +from instances i +left join instance_system_metadata ism_agent + on ism_agent.instance_uuid = i.uuid + and ism_agent.key = "image_hw_qemu_guest_agent" +left join instance_system_metadata ism_user + on ism_user.instance_uuid = i.uuid + and ism_user.key = "image_os_admin_user" +where i.deleted is null + and ism_user.key is null; +```
\ No newline at end of file diff --git a/Updating the image os admin user to fix password resets.md b/Updating the image os admin user to fix password resets.md new file mode 100644 index 0000000..48852de --- /dev/null +++ b/Updating the image os admin user to fix password resets.md @@ -0,0 +1,77 @@ +--- +tags: + - work + - nova +--- + +**instances** +``` ++-------------+-----------------------+------+-----+---------+----------------+ +| Field | Type | Null | Key | Default | Extra | ++-------------+-----------------------+------+-----+---------+----------------+ +| deleted_at | datetime | YES | | NULL | | +| uuid | varchar(36) | NO | UNI | NULL | | ++-------------+-----------------------+------+-----+---------+----------------+ +``` +**instance_system_metadata** +``` ++---------------+--------------+------+-----+---------+----------------+ +| Field | Type | Null | Key | Default | Extra | ++---------------+--------------+------+-----+---------+----------------+ +| created_at | datetime | YES | | NULL | | +| updated_at | datetime | YES | | NULL | | +| deleted_at | datetime | YES | | NULL | | +| id | int(11) | NO | PRI | NULL | auto_increment | +| instance_uuid | varchar(36) | NO | MUL | NULL | | +| key | varchar(255) | NO | | NULL | | +| value | varchar(255) | YES | | NULL | | +| deleted | int(11) | YES | | NULL | | ++---------------+--------------+------+-----+---------+----------------+ +``` + +**Insert new record** +``` +insert into instance_system_metadata (created_at, instance_uuid, `key`, `value`) values (NOW(), "4e675ad6-f01a-476f-8827-8300e540447c", "image_os_admin_user", "administrator" +``` + +**Updated existing values** +``` +update instance_system_metadata set `value` = "administrator", updated_at = NOW() where deleted_at is NULL and `key` = "image_os_admin_user" and instance_uuid in (select instance_uuid from instance_system_metadata where deleted_at is NULL and `key` = "image_os_t +ype" and `value` = "linux"); +``` + +**Which instances currently do not have image_os_admin_user** +``` +select i.uuid from instances i left join instance_system_metadata ism on ism.instance_uuid = i.uuid and ism.key = "image_os_admin_user" where i.deleted_at is null and ism.id is null +``` + +**Which instances of those above, do have qemu guest agent stuff set** +``` +select * from instance_system_metadata ism1 where ism1.instance_uuid in (select i.uuid from instances i left join instance_system_metadata ism on ism.instance_uuid = i.uuid and ism.key = "image_os_admin_user" where i.deleted_at is null and ism.id is null) and ism1.key = "image_hw_qemu_guest_agent"; +``` + +**Insert the image_os_admin_user for those records that miss it but do have qemu guest agent metadata** +``` +INSERT INTO instance_system_metadata (created_at, instance_uuid, `key`, `value`, deleted) +SELECT + NOW(), -- Current timestamp for created_at + i.uuid, -- The UUID of the instance needing the new key + 'image_os_admin_user', -- The fixed key you want to insert + 'administrator', -- The fixed value for the new key + 0 +FROM + instances i +INNER JOIN -- Use INNER JOIN to ensure the instance MUST have the guest agent key + instance_system_metadata ism_agent ON ism_agent.instance_uuid = i.uuid + AND ism_agent.key = 'image_hw_qemu_guest_agent' +LEFT JOIN -- Use LEFT JOIN to check for the PRESENCE of the admin user key + instance_system_metadata ism_admin ON ism_admin.instance_uuid = i.uuid + AND ism_admin.key = 'image_os_admin_user' +WHERE + i.deleted_at IS NULL -- Ensure the instance is not deleted + AND ism_admin.instance_uuid IS NULL; -- This is the crucial part: Select only rows where the LEFT JOIN did NOT find an existing 'image_os_admin_user' key +``` + +``` +insert into instance_system_metadata (created_at, instance_uuid, `key`, `value`, deleted) select NOW(), i.uuid, 'image_os_admin_user', 'administrator', 0 from instances i inner join instance_system_metadata ism_agent on ism_agent.instance_uuid = i.uuid and ism_agent.key = 'image_hw_qemu_guest_agent' left join instance_system_metadata ism_admin on ism_admin.instance_uuid = i.uuid and ism_admin.key = 'image_os_admin_user' where i.deleted_at is null and ism_admin.instance_uuid is null; +```
\ No newline at end of file diff --git a/Using Libvirt to manage QEMU image bitmaps.md b/Using Libvirt to manage QEMU image bitmaps.md new file mode 100644 index 0000000..92eef5f --- /dev/null +++ b/Using Libvirt to manage QEMU image bitmaps.md @@ -0,0 +1,21 @@ +--- +tags: + - qemu + - libvirt +--- +To query info about block devices, including bitmaps we can use a QMP command: +``` +sudo virsh qemu-monitor-command --pretty instance-00026133 '{"execute": "query-block", "arguments": {}}' +``` + +There are six primary bitmap-management API commands: +- `block-dirty-bitmap-add` +- `block-dirty-bitmap-remove` +- `block-dirty-bitmap-clear` +- `block-dirty-bitmap-disable` +- `block-dirty-bitmap-enable` +- `block-dirty-bitmap-merge` +And one related query command: +- `query-block` + +The `node` argument to bitmap cmds can be found as `node-name` in `query-block` results.
\ No newline at end of file |