From 691d635de907755566a290c889206b93a590bc42 Mon Sep 17 00:00:00 2001 From: Jasper Ras Date: Sat, 26 Apr 2025 13:23:57 +0200 Subject: vault backup: 2025-04-26 13:23:57 --- .obsidian/workspace.json | 27 +++++++++--------- daily/10-Apr-2025.md | 18 ------------ daily/11-Apr-2025.md | 27 ------------------ daily/14-Apr-2025.md | 22 -------------- daily/15-Apr-2025.md | 28 ------------------ daily/16-Apr-2025.md | 15 ---------- daily/17-Apr-2025.md | 2 -- daily/18-Apr-2025.md | 10 ------- daily/22-Apr-2025.md | 32 --------------------- daily/23-Apr-2025.md | 68 -------------------------------------------- daily/24-Apr-2025.md | 7 ----- daily/25-Apr-2025.md | 17 ----------- daily/archive/10-Apr-2025.md | 18 ++++++++++++ daily/archive/11-Apr-2025.md | 27 ++++++++++++++++++ daily/archive/14-Apr-2025.md | 22 ++++++++++++++ daily/archive/15-Apr-2025.md | 28 ++++++++++++++++++ daily/archive/16-Apr-2025.md | 15 ++++++++++ daily/archive/17-Apr-2025.md | 2 ++ daily/archive/18-Apr-2025.md | 10 +++++++ daily/archive/22-Apr-2025.md | 32 +++++++++++++++++++++ daily/archive/23-Apr-2025.md | 68 ++++++++++++++++++++++++++++++++++++++++++++ daily/archive/24-Apr-2025.md | 7 +++++ daily/archive/25-Apr-2025.md | 17 +++++++++++ weekly/2025 W17.md | 14 +++++++++ 24 files changed, 274 insertions(+), 259 deletions(-) delete mode 100644 daily/10-Apr-2025.md delete mode 100644 daily/11-Apr-2025.md delete mode 100644 daily/14-Apr-2025.md delete mode 100644 daily/15-Apr-2025.md delete mode 100644 daily/16-Apr-2025.md delete mode 100644 daily/17-Apr-2025.md delete mode 100644 daily/18-Apr-2025.md delete mode 100644 daily/22-Apr-2025.md delete mode 100644 daily/23-Apr-2025.md delete mode 100644 daily/24-Apr-2025.md delete mode 100644 daily/25-Apr-2025.md create mode 100644 daily/archive/10-Apr-2025.md create mode 100644 daily/archive/11-Apr-2025.md create mode 100644 daily/archive/14-Apr-2025.md create mode 100644 daily/archive/15-Apr-2025.md create mode 100644 daily/archive/16-Apr-2025.md create mode 100644 daily/archive/17-Apr-2025.md create mode 100644 daily/archive/18-Apr-2025.md create mode 100644 daily/archive/22-Apr-2025.md create mode 100644 daily/archive/23-Apr-2025.md create mode 100644 daily/archive/24-Apr-2025.md create mode 100644 daily/archive/25-Apr-2025.md create mode 100644 weekly/2025 W17.md diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index bf1f3da..6016eeb 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -13,7 +13,7 @@ "state": { "type": "markdown", "state": { - "file": "Cleaning the Nix store.md", + "file": "daily/archive/22-Apr-2025.md", "mode": "source", "source": false, "backlinks": true, @@ -28,7 +28,7 @@ } }, "icon": "lucide-file", - "title": "Cleaning the Nix store" + "title": "22-Apr-2025" } } ] @@ -185,17 +185,24 @@ "obsidian-excalidraw-plugin:New drawing": false } }, - "active": "17cd99f144675d66", + "active": "f1111e0bbe22e125", "lastOpenFiles": [ + "daily/archive/23-Apr-2025.md", + "daily/archive/24-Apr-2025.md", + "daily/archive/25-Apr-2025.md", + "weekly/2025 W17.md", + "daily/archive/22-Apr-2025.md", + "daily/archive/18-Apr-2025.md", + "daily/archive/17-Apr-2025.md", + "weekly/2025 W16.md", + "daily/archive/10-Apr-2025.md", + "daily/archive/14-Apr-2025.md", + "daily/archive", "Cleaning the Nix store.md", "A nice way to test flake output.md", "A reason for becoming senior is that when Rutger is off I take most of his responsibilities.md", "A useful Go oneliner to run a fileserver.md", "conflict-files-obsidian-git.md", - "daily/25-Apr-2025.md", - "daily/24-Apr-2025.md", - "daily/23-Apr-2025.md", - "daily/22-Apr-2025.md", "ceilometer data is going into Prometheus as Ihor fixed the out of order issue.md", "Using JQ to construct an array and filtering empty strings.md", "Use ssh -O exit user@host to exit a control master session.md", @@ -207,12 +214,6 @@ "Income tax statement and reductions from mortgage.md", "Hypotheekrenteaftrek.md", "GroupVPS Platform.md", - "Enable logging with Libvirt.md", - "Bash process substition (tmp file).md", - "Ansible ad-hoc commands.md", - "Excalidraw/Drawing 2025-04-26 13.13.29.excalidraw.md", - "Building Go applications or packages with Nix.md", - "Specifying a License in a Flake output.md", "Untitled.canvas", "weekly", "archive", diff --git a/daily/10-Apr-2025.md b/daily/10-Apr-2025.md deleted file mode 100644 index 8ac3187..0000000 --- a/daily/10-Apr-2025.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -tags: - - self - - reflection ---- -### I assumed that Mohammed made an oopsie but instead it turned out to be one of us that forgot to clean up. -Today I found out that on the testpod the user sanoid's ssh keys suddenly belonged to Mohammeds' user. I right away went to his chat with the idea that he did something silly probably and even told him that it made me a bit worried. - -I also went to Rutger, who immediately pointed out that it is probably due to us changing the uid of sanoids' user. Which turned out to be true. - -I ask myself the following: why am I quick to jump to a conclusion like "Ah, mohammed might've caused damage by accidentally chowning to much or something like that." -How can I stop myself from that? -I think one way is to ALWAYS force myself to investigate completely, not right away talking about it with others. Keep it to myself, until I really MUST communicate about it. - - -#### Erik doesn't show up at the office for Carlos even though he said that he would be there on Thursday -This kind of triggers a feeling that Erik has a bit of a lax attitude, which often was associated with ops back in the day as well. -He is the designated mentor but I feel he doesn't prepare it well and just "goes with the flow" too much. \ No newline at end of file diff --git a/daily/11-Apr-2025.md b/daily/11-Apr-2025.md deleted file mode 100644 index 8b0a2cf..0000000 --- a/daily/11-Apr-2025.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -tags: - - weekly ---- -This week: -- [[10-Apr-2025]] - -Today marks another Friday, almost weekend, woohoo. - -This week few notable things happened. - -First of all I finally have restored a Ceph backed volume successfully! The issue was actually kind of silly, I forgot to close the read end of a pipe, thus it kept blocking. Luckily I found it, and fixing it was rather trivial. -After that I refactored a bit, to make it little bit better (still not great), and deployed. -I also fixed the request ID logging that was broken for a while, during refactoring of the agent RPC handler I accidentally moved the ctx.update_store outside of the child thread, so the update was useless. Moving it back into the child thread gave back our precious request ids. -Oh and I also found the cause of some sporadic mysql "object belongs to a different session" issue in the backup service that was haunting me. This happened due the Unit Of Work being instantiated only during application startup, specifically for the RPC handler, and then every RPC request used that same UOW. -Because the UOW creates a new session everytime it wasn't completely broken, but occasionally two RPC calls could come in at the same time and then the latter would override the session of the first. -I fixed this by instantiating a UOW per request, this also happens in the API, and is actually the correct way of using it. - -Then I have kicked off the [[List of tags I use in this Vault and their purpose]] note, which contains a list of tags I use within this vault so I don't forget. -As with the current note i'm writing I added the new "weekly" tag to indicate that this "daily" note is actually a week report which I want to write every friday from now on. - -Just had a little brainfart writing the above.. Is it too long? I plan to use these weekly notes to introspect during self assessments, but of course they shouldn't be too tedious to go through... Hmm, well, I guess we'll have to actually **use** it before deciding that. - -I think it will be good practice to link to the current weeks' notes in this weekly note as well so.. see the top :) I probably should put more stuff into daily notes, and then make this a bit of a TLDR. - -I have also been doing some thinking and note taking about [[TDD]] because I feel kind of bad about the current state of the backup service & agent (no tests). We've been looking at [[OpenStack Tempest]] for a bit which is interesting, but I would also like to just create better and more unit tests, especially during development. [[High Gear Low Gear Testing]] was a phrase from the cosmic python book that particularly seemed to resonate with me, but I haven't yet been able to practice it. - diff --git a/daily/14-Apr-2025.md b/daily/14-Apr-2025.md deleted file mode 100644 index 94bc485..0000000 --- a/daily/14-Apr-2025.md +++ /dev/null @@ -1,22 +0,0 @@ -Monday! - -# Standup -OnFailure handlers installeren voor rename script ZFS datasets op backup nodes. -Survey invullen -Puppetrun failed nalopen -Uitzoeken waarom de F QEMU GA sporadisch faalt -- Dit had geloof ik iets te maken met QEMU crash ? Ff dubbel checken, heb er ergens een note van denk ik. -# QA with Webglobe team -Q: Virtual buses, volumes, suggested virtio. Can we do iSCSI instead of VirtIO because we do discards? -A: We don't support discard. NetAPP implements by sending nul bytes. -> increase IO -^ i wouldn't know this - -Q: Can we install from CD? -R: yes, possible. create image, props, iso boot, boot vm rescue from image. -J: documented? -R: Will find dgoc for onehome - -Q: IP addresses, do we really need to let OS handle the allocation? -A: OpenStack does this out of the box; used as single source of truth. -J: Finds reason acceptable; will rewrite. - diff --git a/daily/15-Apr-2025.md b/daily/15-Apr-2025.md deleted file mode 100644 index 06ab31a..0000000 --- a/daily/15-Apr-2025.md +++ /dev/null @@ -1,28 +0,0 @@ -**Interview Ali** - -Q: -Waarom ga je weg bij Leaseplan? -Over openstack deployment: hoe geautomatiseerd; welke tools? - -> Implemented virtual staging clusters mirroring the production architecture using KVM, libvirt, Linux bridge/virtual interfaces, and iptables, reducing setup time by 90% while significantly optimizing costs. - -Did you directly integrate to KVM/Libvirt, can you tell a bit about that? - -A: -php/wordpress dev -exp with cpanel and such -then switch devops - - -+1 kolla ansible / openstack -+1 cpanel/hosting --1 geen puppet ervaring -+1 ovn ervaring; meest voorname probleem is met ovs/ovn - -beetje rare sidetrack config mgmt ansible vs puppet; maakt punt over consistency - -Q: regular day -Q: expectations - -In amsterdam -Avail: 1 june \ No newline at end of file diff --git a/daily/16-Apr-2025.md b/daily/16-Apr-2025.md deleted file mode 100644 index 2d22b55..0000000 --- a/daily/16-Apr-2025.md +++ /dev/null @@ -1,15 +0,0 @@ -1op1 rutger: niet echt iets besproken. - -***Interview Prep Isabel*** -*Do you live in Amsterdam? If yes, how long? plans to stay?* - -*Very shortly worked for ING (2025 january until now). What happened?* - -*At Civir you mention "deployent & administration of cloud technologies" including openstack, does this mean you deplmoyed an openstack cloud or were you a user of an openstack deployment?* - - -*You mention "24/7 support for troubleshooting issues" on multiple positions. What kind of issues?* - -***Isabel Q to us*** - -**Isabel kwam niet opdagen** \ No newline at end of file diff --git a/daily/17-Apr-2025.md b/daily/17-Apr-2025.md deleted file mode 100644 index 7a0a296..0000000 --- a/daily/17-Apr-2025.md +++ /dev/null @@ -1,2 +0,0 @@ -`puppet-neutron` gemerged; gezeik met updaten dependency, uiteindelijk commit hash in lock geupdate. Zie [[Debugging issues with updating Puppet dependency]]. - diff --git a/daily/18-Apr-2025.md b/daily/18-Apr-2025.md deleted file mode 100644 index 7105132..0000000 --- a/daily/18-Apr-2025.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -tags: [] ---- -# Today -Encountered oom on lxchosts. Turned out that octavia wsgi was using huge amounts of ram. -First we disabled the apache on all octavia to prevent more oom kills. -After that Erik limited their allowed memory usage, and we turned them back on. -Found out that we can see what script is ran by apache in the vhost config. Turned out to be some CGI script. -To profile the memory usage I stopped apache and ran a memory profiler directly against the cgi script, I had to stop the LB from using TLS but apart from that it worked smoothly. -We now have a flamegraph of the memory usage, and it looks like it is something to do with ovs. \ No newline at end of file diff --git a/daily/22-Apr-2025.md b/daily/22-Apr-2025.md deleted file mode 100644 index 192eebe..0000000 --- a/daily/22-Apr-2025.md +++ /dev/null @@ -1,32 +0,0 @@ -octavia ovn provider memory leak; found [bug report](https://bugs.launchpad.net/neutron/+bug/2065460) which looks very promising. Trying to patch the driver with this patchset to see if we can fix it. - - -# Interview Ali met Erik -Maand opzegtermijn, geen haast. - -Noemt control plane services -- keystone users projects -werkt voor leaseplan, reason layoff - -migratie workflow: -- legacy cluster - -Live migration, ceph shared, kan geen manier vinden om storage zonder intermediate host te migreren. - -Knows how live migration work. - -Explanation OpenStack, trace server create call: -- keystone, service catalog, token -- nova api, nova scheduler, nova conductor - - doesnt know individual, whole schedules server - - nova libvirt talks libvirt, creates vm - - host aggregates mentioned + flavor extra specs - - vm calls metadata @ 169.169... mentioned - - cloud-init -- nova wants port -> rabbit -> neutron -- neutron api, ovn controller - - ovn northd, nb, sb - - neutron ml2 plugin translates neutron to ovn nb - - northd translates nb to sb - - ovn controller reads sb and translate to ovs on compute -- glance image \ No newline at end of file diff --git a/daily/23-Apr-2025.md b/daily/23-Apr-2025.md deleted file mode 100644 index 6c9b6c6..0000000 --- a/daily/23-Apr-2025.md +++ /dev/null @@ -1,68 +0,0 @@ - -# Interview Isabel -devops engineer 7yrs - -Provides maintainance os cloud - -exp with openstack: - - deploying new compute - - maintaining - - remember rabbitmq incident: queueing - - not used puppet - - ansible -exp with openstack: - - deploying new compute - - maintaining - - remember rabbitmq incident: queueing - - not used puppet - - ansible -#### Our questions -How are you in programming? -- Really like it, create many tools -- Python / ansible deploy infra automated instead of manually - - *Realised manual labor and automated it* - -How do you feel about going more into a development role? -- that's what im looking for, prefer to be making. - -Linux or windows experience? -- check fs -- processes -- administrative -- k8s many scripts - -How would you solve a problem where a VM is not starting? -- Check nova compute for error -- If ceilometer/logging check that -- Try with nova-compute to restart if down -- Reload instance (?) - -Have you ever had to go into openstack DBs? -- Not really, -- Do have SQL knowledge - -Do you know how to work with git? -- yes, branch system current job - -**Why are you leaving your current position?** -reason: different tech than banks -same company HCL as in spain, change contract - - -*Deployment, maintenance, and administration of cloud technologies -VMware, Azure, Openstack.* -**Does this mean workloads running on said clouds? Or does this also apply to managing infrastructure such as openstack?** - -*Bash scripting for Linux server automation.* **What sort of automation?** - -*Plan and execute migrations and patching from on-premises infrastructure -to ING Private cloud (IPC)* **Can you talk more about what kind of migrations** - -#### Isabel questions to us - -What does the usage of openstack look like from customer perspective? -- different kinds (brands, direct access) -Own DC? yes -Are you expecting me to create new components, or maintain? -- maintain, puppet etc, - diff --git a/daily/24-Apr-2025.md b/daily/24-Apr-2025.md deleted file mode 100644 index 371dd15..0000000 --- a/daily/24-Apr-2025.md +++ /dev/null @@ -1,7 +0,0 @@ -Ceilometer thaw/freeze gecheckt: linear vs unordered flow. Geen metrics testpod -> prod deploy - -Octavia / system test cph8 onderzocht: kon netwerk niet vinden in ovn. Restart resolved issue. - -OVN database cluster aanmerken in ansible ipv alle netwerk nodes zien als database hosts. Testen op testpod. - -Lijst controleren van backup contracten vs gobs. 170 active contracten vs 191 periodic backups waarvan er 44 disabled. \ No newline at end of file diff --git a/daily/25-Apr-2025.md b/daily/25-Apr-2025.md deleted file mode 100644 index 9514ec3..0000000 --- a/daily/25-Apr-2025.md +++ /dev/null @@ -1,17 +0,0 @@ -Gobs OSC plugin pagination -Deploy goba with ceilometer thaw/freeze to prod - -# Tech interview Isabel -Intro -- [ ] What we will do: some questions then workshop -- [ ] Erik intro -- [ ] Isabel intro - -Questions: -- [ ] What is your experience with `git`, are you comfortable with it? -- [ ] Can you expand a little bit on your programming experiences, what are some examples of projects that you worked on? -- [ ] What did you do with Ansible? -- [ ] Can you talk a little about openstack, what is it and what are the most core components? -- [ ] Workshop - -Isabel questions: diff --git a/daily/archive/10-Apr-2025.md b/daily/archive/10-Apr-2025.md new file mode 100644 index 0000000..8ac3187 --- /dev/null +++ b/daily/archive/10-Apr-2025.md @@ -0,0 +1,18 @@ +--- +tags: + - self + - reflection +--- +### I assumed that Mohammed made an oopsie but instead it turned out to be one of us that forgot to clean up. +Today I found out that on the testpod the user sanoid's ssh keys suddenly belonged to Mohammeds' user. I right away went to his chat with the idea that he did something silly probably and even told him that it made me a bit worried. + +I also went to Rutger, who immediately pointed out that it is probably due to us changing the uid of sanoids' user. Which turned out to be true. + +I ask myself the following: why am I quick to jump to a conclusion like "Ah, mohammed might've caused damage by accidentally chowning to much or something like that." +How can I stop myself from that? +I think one way is to ALWAYS force myself to investigate completely, not right away talking about it with others. Keep it to myself, until I really MUST communicate about it. + + +#### Erik doesn't show up at the office for Carlos even though he said that he would be there on Thursday +This kind of triggers a feeling that Erik has a bit of a lax attitude, which often was associated with ops back in the day as well. +He is the designated mentor but I feel he doesn't prepare it well and just "goes with the flow" too much. \ No newline at end of file diff --git a/daily/archive/11-Apr-2025.md b/daily/archive/11-Apr-2025.md new file mode 100644 index 0000000..8b0a2cf --- /dev/null +++ b/daily/archive/11-Apr-2025.md @@ -0,0 +1,27 @@ +--- +tags: + - weekly +--- +This week: +- [[10-Apr-2025]] + +Today marks another Friday, almost weekend, woohoo. + +This week few notable things happened. + +First of all I finally have restored a Ceph backed volume successfully! The issue was actually kind of silly, I forgot to close the read end of a pipe, thus it kept blocking. Luckily I found it, and fixing it was rather trivial. +After that I refactored a bit, to make it little bit better (still not great), and deployed. +I also fixed the request ID logging that was broken for a while, during refactoring of the agent RPC handler I accidentally moved the ctx.update_store outside of the child thread, so the update was useless. Moving it back into the child thread gave back our precious request ids. +Oh and I also found the cause of some sporadic mysql "object belongs to a different session" issue in the backup service that was haunting me. This happened due the Unit Of Work being instantiated only during application startup, specifically for the RPC handler, and then every RPC request used that same UOW. +Because the UOW creates a new session everytime it wasn't completely broken, but occasionally two RPC calls could come in at the same time and then the latter would override the session of the first. +I fixed this by instantiating a UOW per request, this also happens in the API, and is actually the correct way of using it. + +Then I have kicked off the [[List of tags I use in this Vault and their purpose]] note, which contains a list of tags I use within this vault so I don't forget. +As with the current note i'm writing I added the new "weekly" tag to indicate that this "daily" note is actually a week report which I want to write every friday from now on. + +Just had a little brainfart writing the above.. Is it too long? I plan to use these weekly notes to introspect during self assessments, but of course they shouldn't be too tedious to go through... Hmm, well, I guess we'll have to actually **use** it before deciding that. + +I think it will be good practice to link to the current weeks' notes in this weekly note as well so.. see the top :) I probably should put more stuff into daily notes, and then make this a bit of a TLDR. + +I have also been doing some thinking and note taking about [[TDD]] because I feel kind of bad about the current state of the backup service & agent (no tests). We've been looking at [[OpenStack Tempest]] for a bit which is interesting, but I would also like to just create better and more unit tests, especially during development. [[High Gear Low Gear Testing]] was a phrase from the cosmic python book that particularly seemed to resonate with me, but I haven't yet been able to practice it. + diff --git a/daily/archive/14-Apr-2025.md b/daily/archive/14-Apr-2025.md new file mode 100644 index 0000000..f0209e8 --- /dev/null +++ b/daily/archive/14-Apr-2025.md @@ -0,0 +1,22 @@ +Monday! + +# Standup +OnFailure handlers installeren voor rename script ZFS datasets op backup nodes. +Survey invullen +Puppetrun failed nalopen +Uitzoeken waarom de F QEMU GA sporadisch faalt +- Dit had geloof ik iets te maken met QEMU crash ? Ff dubbel checken, heb er ergens een note van denk ik. +# QA with Webglobe team +Q: Virtual buses, volumes, suggested virtio. Can we do iSCSI instead of VirtIO because we do discards? +A: We don't support discard. NetAPP implements by sending nul bytes. -> increase IO +^ i wouldn't know this + +Q: Can we install from CD? +R: yes, possible. create image, props, iso boot, boot vm rescue from image. +J: documented? +R: Will find dgoc for onehome + +Q: IP addresses, do we really need to let OS handle the allocation? +A: OpenStack does this out of the box; used as single source of tru. +J: Finds reason acceptable; will rewrite.th + diff --git a/daily/archive/15-Apr-2025.md b/daily/archive/15-Apr-2025.md new file mode 100644 index 0000000..06ab31a --- /dev/null +++ b/daily/archive/15-Apr-2025.md @@ -0,0 +1,28 @@ +**Interview Ali** + +Q: +Waarom ga je weg bij Leaseplan? +Over openstack deployment: hoe geautomatiseerd; welke tools? + +> Implemented virtual staging clusters mirroring the production architecture using KVM, libvirt, Linux bridge/virtual interfaces, and iptables, reducing setup time by 90% while significantly optimizing costs. + +Did you directly integrate to KVM/Libvirt, can you tell a bit about that? + +A: +php/wordpress dev +exp with cpanel and such +then switch devops + + ++1 kolla ansible / openstack ++1 cpanel/hosting +-1 geen puppet ervaring ++1 ovn ervaring; meest voorname probleem is met ovs/ovn + +beetje rare sidetrack config mgmt ansible vs puppet; maakt punt over consistency + +Q: regular day +Q: expectations + +In amsterdam +Avail: 1 june \ No newline at end of file diff --git a/daily/archive/16-Apr-2025.md b/daily/archive/16-Apr-2025.md new file mode 100644 index 0000000..2d22b55 --- /dev/null +++ b/daily/archive/16-Apr-2025.md @@ -0,0 +1,15 @@ +1op1 rutger: niet echt iets besproken. + +***Interview Prep Isabel*** +*Do you live in Amsterdam? If yes, how long? plans to stay?* + +*Very shortly worked for ING (2025 january until now). What happened?* + +*At Civir you mention "deployent & administration of cloud technologies" including openstack, does this mean you deplmoyed an openstack cloud or were you a user of an openstack deployment?* + + +*You mention "24/7 support for troubleshooting issues" on multiple positions. What kind of issues?* + +***Isabel Q to us*** + +**Isabel kwam niet opdagen** \ No newline at end of file diff --git a/daily/archive/17-Apr-2025.md b/daily/archive/17-Apr-2025.md new file mode 100644 index 0000000..7a0a296 --- /dev/null +++ b/daily/archive/17-Apr-2025.md @@ -0,0 +1,2 @@ +`puppet-neutron` gemerged; gezeik met updaten dependency, uiteindelijk commit hash in lock geupdate. Zie [[Debugging issues with updating Puppet dependency]]. + diff --git a/daily/archive/18-Apr-2025.md b/daily/archive/18-Apr-2025.md new file mode 100644 index 0000000..7105132 --- /dev/null +++ b/daily/archive/18-Apr-2025.md @@ -0,0 +1,10 @@ +--- +tags: [] +--- +# Today +Encountered oom on lxchosts. Turned out that octavia wsgi was using huge amounts of ram. +First we disabled the apache on all octavia to prevent more oom kills. +After that Erik limited their allowed memory usage, and we turned them back on. +Found out that we can see what script is ran by apache in the vhost config. Turned out to be some CGI script. +To profile the memory usage I stopped apache and ran a memory profiler directly against the cgi script, I had to stop the LB from using TLS but apart from that it worked smoothly. +We now have a flamegraph of the memory usage, and it looks like it is something to do with ovs. \ No newline at end of file diff --git a/daily/archive/22-Apr-2025.md b/daily/archive/22-Apr-2025.md new file mode 100644 index 0000000..192eebe --- /dev/null +++ b/daily/archive/22-Apr-2025.md @@ -0,0 +1,32 @@ +octavia ovn provider memory leak; found [bug report](https://bugs.launchpad.net/neutron/+bug/2065460) which looks very promising. Trying to patch the driver with this patchset to see if we can fix it. + + +# Interview Ali met Erik +Maand opzegtermijn, geen haast. + +Noemt control plane services +- keystone users projects +werkt voor leaseplan, reason layoff + +migratie workflow: +- legacy cluster + +Live migration, ceph shared, kan geen manier vinden om storage zonder intermediate host te migreren. + +Knows how live migration work. + +Explanation OpenStack, trace server create call: +- keystone, service catalog, token +- nova api, nova scheduler, nova conductor + - doesnt know individual, whole schedules server + - nova libvirt talks libvirt, creates vm + - host aggregates mentioned + flavor extra specs + - vm calls metadata @ 169.169... mentioned + - cloud-init +- nova wants port -> rabbit -> neutron +- neutron api, ovn controller + - ovn northd, nb, sb + - neutron ml2 plugin translates neutron to ovn nb + - northd translates nb to sb + - ovn controller reads sb and translate to ovs on compute +- glance image \ No newline at end of file diff --git a/daily/archive/23-Apr-2025.md b/daily/archive/23-Apr-2025.md new file mode 100644 index 0000000..6c9b6c6 --- /dev/null +++ b/daily/archive/23-Apr-2025.md @@ -0,0 +1,68 @@ + +# Interview Isabel +devops engineer 7yrs + +Provides maintainance os cloud + +exp with openstack: + - deploying new compute + - maintaining + - remember rabbitmq incident: queueing + - not used puppet + - ansible +exp with openstack: + - deploying new compute + - maintaining + - remember rabbitmq incident: queueing + - not used puppet + - ansible +#### Our questions +How are you in programming? +- Really like it, create many tools +- Python / ansible deploy infra automated instead of manually + - *Realised manual labor and automated it* + +How do you feel about going more into a development role? +- that's what im looking for, prefer to be making. + +Linux or windows experience? +- check fs +- processes +- administrative +- k8s many scripts + +How would you solve a problem where a VM is not starting? +- Check nova compute for error +- If ceilometer/logging check that +- Try with nova-compute to restart if down +- Reload instance (?) + +Have you ever had to go into openstack DBs? +- Not really, +- Do have SQL knowledge + +Do you know how to work with git? +- yes, branch system current job + +**Why are you leaving your current position?** +reason: different tech than banks +same company HCL as in spain, change contract + + +*Deployment, maintenance, and administration of cloud technologies +VMware, Azure, Openstack.* +**Does this mean workloads running on said clouds? Or does this also apply to managing infrastructure such as openstack?** + +*Bash scripting for Linux server automation.* **What sort of automation?** + +*Plan and execute migrations and patching from on-premises infrastructure +to ING Private cloud (IPC)* **Can you talk more about what kind of migrations** + +#### Isabel questions to us + +What does the usage of openstack look like from customer perspective? +- different kinds (brands, direct access) +Own DC? yes +Are you expecting me to create new components, or maintain? +- maintain, puppet etc, + diff --git a/daily/archive/24-Apr-2025.md b/daily/archive/24-Apr-2025.md new file mode 100644 index 0000000..371dd15 --- /dev/null +++ b/daily/archive/24-Apr-2025.md @@ -0,0 +1,7 @@ +Ceilometer thaw/freeze gecheckt: linear vs unordered flow. Geen metrics testpod -> prod deploy + +Octavia / system test cph8 onderzocht: kon netwerk niet vinden in ovn. Restart resolved issue. + +OVN database cluster aanmerken in ansible ipv alle netwerk nodes zien als database hosts. Testen op testpod. + +Lijst controleren van backup contracten vs gobs. 170 active contracten vs 191 periodic backups waarvan er 44 disabled. \ No newline at end of file diff --git a/daily/archive/25-Apr-2025.md b/daily/archive/25-Apr-2025.md new file mode 100644 index 0000000..9514ec3 --- /dev/null +++ b/daily/archive/25-Apr-2025.md @@ -0,0 +1,17 @@ +Gobs OSC plugin pagination +Deploy goba with ceilometer thaw/freeze to prod + +# Tech interview Isabel +Intro +- [ ] What we will do: some questions then workshop +- [ ] Erik intro +- [ ] Isabel intro + +Questions: +- [ ] What is your experience with `git`, are you comfortable with it? +- [ ] Can you expand a little bit on your programming experiences, what are some examples of projects that you worked on? +- [ ] What did you do with Ansible? +- [ ] Can you talk a little about openstack, what is it and what are the most core components? +- [ ] Workshop + +Isabel questions: diff --git a/weekly/2025 W17.md b/weekly/2025 W17.md new file mode 100644 index 0000000..7cc20b7 --- /dev/null +++ b/weekly/2025 W17.md @@ -0,0 +1,14 @@ +Another week + +- [[22-Apr-2025]] +- [[23-Apr-2025]] +- [[24-Apr-2025]] +- [[25-Apr-2025]] + +Did some minor chores and bug fixes. +Prepared for OVN upgrade next week. I'm rather anxious about that. +The team also resolved long-standing issues with OVN in CPH8. We made sure all the configs are up-to-date and the cluster members are n01's in vps1/2/3, as we do in other regions. Hopefully this makes the upgrade go well. + +We conducted more interviews as well. The second (technical) interview with Ali. It went well and we want to make him an offer. +Then we had a first interview with Isabel which was also nice, we figured that she could join instead of Carlos whom we had to let go. +Sadly the second interview that Friday she didn't show up - for the second time. So we'll pass. \ No newline at end of file -- cgit v1.2.3