Update infrastructure

infrastructure.org

@@ -1,123 +1,99 @@
-#+TITLE: Creating AWS Infrastructure for Common Lisp Web-Application using cloudinit && terraform
+#+title: IaC AWS for ml-survey
-#+PROPERTY: header-args:hcl :tangle main.tf :mkdirp yes
+#+author: Marcus Kammer
 
-#+begin_src python :noweb "yes"
+* About
-  import os
-  from collections import OrderedDict
-  import yaml
 
-  def represent_ordereddict(dumper, data):
+This org file is part of the [[https://code.siemens.com/marcus.kammer/ml-survey][ml-survey]] repository. It is meant to be the
-      return dumper.represent_mapping('tag:yaml.org,2002:map', data.items())
+infrastructure-as-code documentation.
 
-  yaml.add_representer(OrderedDict, represent_ordereddict)
-
-  # Define the order of keys
-  key_order = [
-      "local",
-      "keyboard",
-      "timezone",
-      "groups",
-      "users",
-      "packages",
-      "package_update",
-      "package_upgrade"
-  ]
-
-  # Initialize OrderedDict with the defined keys
-  cloudinit_data = OrderedDict((key, None) for key in key_order)
-
-  <<local-settings>>
-  <<timezone-setting>>
-  <<groups-creation>>
-  <<user-creation-configuration>>
-  <<package-install>>
-
-  # Remove any keys with None values
-  cloudinit_data = OrderedDict((k, v) for k, v in cloudinit_data.items() if v is not None)
-
-  with open("cloudinit1.yaml", 'w') as file:
-      file.write("#cloud-config\n")
-      yaml.dump(cloudinit_data, file, default_flow_style=False)
-#+end_src
-
-#+RESULTS:
-: None
 
 * Cloudinit
+:PROPERTIES:
+:header-args:yaml: :tangle cloudinit.yml
+:END:
 ** Introduction
 
 This document explains the structure and content of our ~cloudinit.yml~ file,
 which is used to initialize our AWS EC2 instance. The code blocks in this file
 can be tangled to create the final ~cloudinit.yml~ file.
 
-** Locale and Keyboard Settings
+** Cloud-Config Header
+Every cloud-init file should start with "#cloud-config". This tells cloud-init that the file is a cloud-config file.
 
+#+BEGIN_SRC yaml
+#cloud-config
+#+END_SRC
+
+** Locale and Keyboard Settings
 Set the system locale and keyboard layout.
 
-#+name: local-settings
+#+BEGIN_SRC yaml
-#+begin_src python
+locale: en_US.UTF-8
-  cloudinit_data["local"] = "en_US.UTF-8"
+keyboard:
-  cloudinit_data["keyboard"] = {"layout": "en_US.UTF-8"}
+  layout: us
-#+end_src
+#+END_SRC
 
 ** Timezone Setting
-
 Set the system timezone.
 
-#+name: timezone-setting
+#+BEGIN_SRC yaml
-#+begin_src python
+timezone: Europe/Berlin
-  cloudinit_data["timezone"] = "Europe/Berlin"
+#+END_SRC
-#+end_src
-
-** Groups Creation
 
+** Group Creation
 Create any necessary system groups.
 
-#+name: groups-creation
+#+BEGIN_SRC yaml
-#+begin_src python
+groups:
-  cloudinit_data["groups"] = ["nginxgroup"]
+  - nginxgroup
-#+end_src
+#+END_SRC
 
 ** User Creation and Configuration
+Create and configure users. Here we're creating two users: a system user for Nginx and a regular user for administration.
 
-Create and configure users. Here we're creating two users: a system user for
+#+BEGIN_SRC yaml
-Nginx and a regular user for administration.
+users:
-
+  - name: nginxuser
-#+name: user-creation-configuration
+    system: true
-#+begin_src python
+    shell: /usr/sbin/nologin
-  def read_ssh_key(identity="id_ed25519"):
+    groups: nginxgroup
-      home_dir = os.path.expanduser("~")
+    sudo: null
-      ssh_file_path = os.path.join(home_dir, ".ssh", f"{identity}.pub")
+  - name: cl
-
+    groups: users, admin
-      try:
+    sudo: ALL=(ALL) NOPASSWD:ALL
-          with open(ssh_file_path, "r") as ssh_file:
+    shell: /bin/bash
-              return ssh_file.readline().strip()
+    ssh_authorized_keys:
-      except FileNotFoundError:
+      - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+46Y3AHPLJgz8KK61doqH3jBX2TL3TJvZsJrB9Km03 visua@xps-8930
-          print(f"SSH key file not found: {ssh_file_path}")
+      - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMIHJ5qpMIKL7N3nC0GG1O4ygtkqOlQuZReoik6xGBxn marcus@XPS-13-9380.local
-          return None
+      - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB6xSH5nE0uy0C0kglpp4EqrbbW2CrBeAIj+X6Sf2pd0 XPS-8930-Ubuntu_22
-
+#+END_SRC
-  cloudinit_data["users"] = [{"name": "nginxuser",
-                              "system": "true",
-                              "shell": "/usr/sbin/nologin",
-                              "groups": ["nginxgroup"],
-                              "sudo": "null"},
-                             {"name": "cl",
-                              "groups": ["users", "admin"],
-                              "sudo": "ALL=(ALL) NOPASSWD:ALL",
-                              "shell": "/bin/bash",
-                              "ssh_authorized_keys": [read_ssh_key()]}]
-#+end_src
 
 ** Package Installation
-
 Install necessary packages.
 
-#+name: package-install
+#+BEGIN_SRC yaml
-#+begin_src python :results none
+packages:
-  cloudinit_data["packages"] = ["detachtty", "fail2ban", "ufw", "unattended-upgrades", "sbcl", "mosh", "tmux", "git", "nginx", "certbot", "python3-certbot-nginx", "build-essential", "libzstd-dev", "libsqlite3-dev", "sqlite3", "curl", "wget"]
+  - detachtty
-  cloudinit_data["package_update"] = True
+  - fail2ban
-  cloudinit_data["package_upgrade"] = True
+  - ufw
-#+end_src
+  - unattended-upgrades
+  - sbcl
+  - mosh
+  - tmux
+  - git
+  - nginx
+  - certbot
+  - python3-certbot-nginx
+  - build-essential
+  - libzstd-dev
+  - libsqlite3-dev
+  - sqlite3
+  - curl
+  - wget
+
+package_update: true
+package_upgrade: true
+#+END_SRC
 
 ** File Writing
 
@@ -128,6 +104,7 @@ write_files:
 #+END_SRC
 
 *** Automatic Upgrades Configuration
+
 #+BEGIN_SRC yaml
   - path: /etc/apt/apt.conf.d/20auto-upgrades
     content: |
@@ -138,6 +115,7 @@ write_files:
 #+END_SRC
 
 *** SSH Configuration
+
 #+BEGIN_SRC yaml
   - path: /etc/ssh/sshd_config
     content: |
@@ -163,6 +141,7 @@ write_files:
 #+END_SRC
 
 *** Fail2Ban Configuration
+
 #+BEGIN_SRC yaml
   - path: /etc/fail2ban/jail.local
     content: |
@@ -195,6 +174,7 @@ write_files:
 #+END_SRC
 
 *** Nginx Configuration
+
 #+BEGIN_SRC yaml
   - path: /etc/nginx/nginx.conf
     content: |
@@ -224,6 +204,7 @@ write_files:
 #+END_SRC
 
 *** Nginx Reverse Proxy Configuration
+
 #+BEGIN_SRC yaml
   - path: /etc/nginx/sites-available/reverse-proxy.conf
     content: |
@@ -253,6 +234,7 @@ write_files:
 #+END_SRC
 
 *** Git Configuration Script
+
 #+BEGIN_SRC yaml
   - path: /home/cl/setup_git.sh
     owner: 'cl:cl'
@@ -267,6 +249,7 @@ write_files:
 #+END_SRC
 
 *** Repository Setup Script
+
 #+BEGIN_SRC yaml
   - path: /home/cl/setup_repos.sh
     owner: 'cl:cl'
@@ -278,6 +261,7 @@ write_files:
 #+END_SRC
 
 *** User Setup Script
+
 #+BEGIN_SRC yaml
   - path: /home/cl/setup_user_all.sh
     owner: 'cl:cl'
@@ -292,6 +276,7 @@ write_files:
 #+END_SRC
 
 *** OpenAI Bot Blocking Script
+
 #+BEGIN_SRC yaml
   - path: /home/cl/openai_block_access.sh
     owner: 'cl:cl'
@@ -311,6 +296,7 @@ write_files:
 #+END_SRC
 
 *** Tmux Configuration
+
 #+BEGIN_SRC yaml
   - path: /home/cl/.tmux.conf
     owner: 'cl:cl'
@@ -547,46 +533,6 @@ runcmd:
   - sudo -u cl /bin/bash /home/cl/setup_user_all.sh
 #+END_SRC
 
-** SSL Setup Script
-
-Add a script to set up SSL certificates manually when needed.
-
-#+BEGIN_SRC yaml
-write_files:
-  - path: /home/cl/setup_ssl.sh
-    owner: cl:cl
-    permissions: '0755'
-    content: |
-      #!/bin/bash
-      # This script sets up SSL certificates using Let's Encrypt
-
-      # Check if domain is provided
-      if [ $# -eq 0 ]; then
-          echo "Please provide a domain name as an argument."
-          exit 1
-      fi
-
-      DOMAIN=$1
-
-      # Install certbot if not already installed
-      if ! command -v certbot &> /dev/null; then
-          sudo apt update
-          sudo apt install -y certbot python3-certbot-nginx
-      fi
-
-      # Obtain SSL certificate
-      sudo certbot --nginx -d $DOMAIN --non-interactive --agree-tos --email your-email@example.com --redirect
-
-      # Setup auto-renewal
-      (crontab -l 2>/dev/null; echo "0 12 * * * /usr/bin/certbot renew --quiet") | crontab -
-
-      echo "SSL certificate has been set up for $DOMAIN"
-
-runcmd:
-  # Install certbot
-  - apt update
-  - apt install -y certbot python3-certbot-nginx
-#+END_SRC
 
 ** Conclusion
 This concludes the documentation for our ~cloudinit.yml~ file. To generate the actual YAML file from this Org document, you can use the following Emacs command:
@@ -602,7 +548,12 @@ Or in an Org-mode babel shell block:
 This will create the ~cloudinit.yml~ file with all the code blocks in the correct order and with proper indentation.
 
 Remember to review the generated YAML file to ensure all indentations are correct, as YAML is sensitive to indentation.
+
+
 * Terraform and AWS
+:PROPERTIES:
+:header-args:hcl: :tangle main.tf :mkdirp yes
+:END:
 ** Introduction
 
 This tutorial will guide you through creating a ~main.tf~ file for setting up
@@ -610,6 +561,41 @@ basic AWS infrastructure using Terraform. We'll explain each resource, why it's
 necessary, and the order in which they should be created. The code blocks in
 this file can be tangled to create the final ~main.tf~ file.
 
+#+name: tf-graph
+#+begin_src powershell :results output
+  terraform graph
+#+end_src
+
+#+begin_src dot :var g=tf-graph :file tf-graph.png :exports results
+  $g
+#+end_src
+
+#+RESULTS:
+[[file:tf-graph.png]]
+
+#+name: tf-plan
+#+begin_src powershell :results output :exports none
+  terraform plan
+#+end_src
+
+#+name: tf-destroy
+#+begin_src powershell :results output :exports none
+  terraform destroy -auto-approve
+#+end_src
+
+** Define Global Variables
+
+#+begin_src hcl :tangle variables.tf
+  variable "host_os" {
+    type = string
+    default = "windows"
+  }
+#+end_src
+
+#+begin_src hcl :tangle terraform.tfvars
+  host_os = "windows"
+#+end_src
+
 ** Virtual Private Cloud (VPC)
 
 We start with creating a VPC, which is a virtual network dedicated to your AWS
@@ -778,6 +764,7 @@ This EC2 instance:
 - Uses the cloud-init configuration we specified.
 
 ** Output
+
 Lastly, we add an output to display the public IP of our instance.
 
 #+BEGIN_SRC hcl
@@ -795,8 +782,26 @@ sets up a basic AWS infrastructure. The resources are created in a logical
 order, with each building upon the previous ones to create a fully functional
 network and compute environment in AWS.
 
-Remember to also create the necessary ~datasource.tf~ and ~providers.tf~ files
+#+name: tf-apply
-to complete your Terraform configuration.
+#+begin_src powershell :results output :exports none
+  terraform apply -auto-approve
+#+end_src
 
-To tangle this file and create ~main.tf~, you can use the following Emacs
+
-command: ~C-c C-v t~
+* Web App
+
+Write a tiny web application to test if the infrastructure is successful.
+
+#+begin_src lisp
+  (ql:quickload #:hunchentoot)
+
+  (defpackage sample-web-app
+    (:use #:cl)
+    (:import-from #:hunchentoot
+                  #:define-easy-handler
+                  #:start
+                  #:stop)
+    (:documentation "Sample web app package."))
+
+  (in-package #:sample-web-app)
+#+end_src