diff --git a/infrastructure.org b/infrastructure.org index d1c8f7e..8224f8c 100644 --- a/infrastructure.org +++ b/infrastructure.org @@ -1,123 +1,99 @@ -#+TITLE: Creating AWS Infrastructure for Common Lisp Web-Application using cloudinit && terraform -#+PROPERTY: header-args:hcl :tangle main.tf :mkdirp yes +#+title: IaC AWS for ml-survey +#+author: Marcus Kammer -#+begin_src python :noweb "yes" - import os - from collections import OrderedDict - import yaml +* About - def represent_ordereddict(dumper, data): - return dumper.represent_mapping('tag:yaml.org,2002:map', data.items()) +This org file is part of the [[https://code.siemens.com/marcus.kammer/ml-survey][ml-survey]] repository. It is meant to be the +infrastructure-as-code documentation. - yaml.add_representer(OrderedDict, represent_ordereddict) - - # Define the order of keys - key_order = [ - "local", - "keyboard", - "timezone", - "groups", - "users", - "packages", - "package_update", - "package_upgrade" - ] - - # Initialize OrderedDict with the defined keys - cloudinit_data = OrderedDict((key, None) for key in key_order) - - <> - <> - <> - <> - <> - - # Remove any keys with None values - cloudinit_data = OrderedDict((k, v) for k, v in cloudinit_data.items() if v is not None) - - with open("cloudinit1.yaml", 'w') as file: - file.write("#cloud-config\n") - yaml.dump(cloudinit_data, file, default_flow_style=False) -#+end_src - -#+RESULTS: -: None * Cloudinit +:PROPERTIES: +:header-args:yaml: :tangle cloudinit.yml +:END: ** Introduction This document explains the structure and content of our ~cloudinit.yml~ file, which is used to initialize our AWS EC2 instance. The code blocks in this file can be tangled to create the final ~cloudinit.yml~ file. -** Locale and Keyboard Settings +** Cloud-Config Header +Every cloud-init file should start with "#cloud-config". This tells cloud-init that the file is a cloud-config file. +#+BEGIN_SRC yaml +#cloud-config +#+END_SRC + +** Locale and Keyboard Settings Set the system locale and keyboard layout. -#+name: local-settings -#+begin_src python - cloudinit_data["local"] = "en_US.UTF-8" - cloudinit_data["keyboard"] = {"layout": "en_US.UTF-8"} -#+end_src +#+BEGIN_SRC yaml +locale: en_US.UTF-8 +keyboard: + layout: us +#+END_SRC ** Timezone Setting - Set the system timezone. -#+name: timezone-setting -#+begin_src python - cloudinit_data["timezone"] = "Europe/Berlin" -#+end_src - -** Groups Creation +#+BEGIN_SRC yaml +timezone: Europe/Berlin +#+END_SRC +** Group Creation Create any necessary system groups. -#+name: groups-creation -#+begin_src python - cloudinit_data["groups"] = ["nginxgroup"] -#+end_src +#+BEGIN_SRC yaml +groups: + - nginxgroup +#+END_SRC ** User Creation and Configuration +Create and configure users. Here we're creating two users: a system user for Nginx and a regular user for administration. -Create and configure users. Here we're creating two users: a system user for -Nginx and a regular user for administration. - -#+name: user-creation-configuration -#+begin_src python - def read_ssh_key(identity="id_ed25519"): - home_dir = os.path.expanduser("~") - ssh_file_path = os.path.join(home_dir, ".ssh", f"{identity}.pub") - - try: - with open(ssh_file_path, "r") as ssh_file: - return ssh_file.readline().strip() - except FileNotFoundError: - print(f"SSH key file not found: {ssh_file_path}") - return None - - cloudinit_data["users"] = [{"name": "nginxuser", - "system": "true", - "shell": "/usr/sbin/nologin", - "groups": ["nginxgroup"], - "sudo": "null"}, - {"name": "cl", - "groups": ["users", "admin"], - "sudo": "ALL=(ALL) NOPASSWD:ALL", - "shell": "/bin/bash", - "ssh_authorized_keys": [read_ssh_key()]}] -#+end_src +#+BEGIN_SRC yaml +users: + - name: nginxuser + system: true + shell: /usr/sbin/nologin + groups: nginxgroup + sudo: null + - name: cl + groups: users, admin + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + ssh_authorized_keys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+46Y3AHPLJgz8KK61doqH3jBX2TL3TJvZsJrB9Km03 visua@xps-8930 + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMIHJ5qpMIKL7N3nC0GG1O4ygtkqOlQuZReoik6xGBxn marcus@XPS-13-9380.local + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB6xSH5nE0uy0C0kglpp4EqrbbW2CrBeAIj+X6Sf2pd0 XPS-8930-Ubuntu_22 +#+END_SRC ** Package Installation - Install necessary packages. -#+name: package-install -#+begin_src python :results none - cloudinit_data["packages"] = ["detachtty", "fail2ban", "ufw", "unattended-upgrades", "sbcl", "mosh", "tmux", "git", "nginx", "certbot", "python3-certbot-nginx", "build-essential", "libzstd-dev", "libsqlite3-dev", "sqlite3", "curl", "wget"] - cloudinit_data["package_update"] = True - cloudinit_data["package_upgrade"] = True -#+end_src +#+BEGIN_SRC yaml +packages: + - detachtty + - fail2ban + - ufw + - unattended-upgrades + - sbcl + - mosh + - tmux + - git + - nginx + - certbot + - python3-certbot-nginx + - build-essential + - libzstd-dev + - libsqlite3-dev + - sqlite3 + - curl + - wget + +package_update: true +package_upgrade: true +#+END_SRC ** File Writing @@ -128,6 +104,7 @@ write_files: #+END_SRC *** Automatic Upgrades Configuration + #+BEGIN_SRC yaml - path: /etc/apt/apt.conf.d/20auto-upgrades content: | @@ -138,6 +115,7 @@ write_files: #+END_SRC *** SSH Configuration + #+BEGIN_SRC yaml - path: /etc/ssh/sshd_config content: | @@ -163,6 +141,7 @@ write_files: #+END_SRC *** Fail2Ban Configuration + #+BEGIN_SRC yaml - path: /etc/fail2ban/jail.local content: | @@ -195,6 +174,7 @@ write_files: #+END_SRC *** Nginx Configuration + #+BEGIN_SRC yaml - path: /etc/nginx/nginx.conf content: | @@ -224,6 +204,7 @@ write_files: #+END_SRC *** Nginx Reverse Proxy Configuration + #+BEGIN_SRC yaml - path: /etc/nginx/sites-available/reverse-proxy.conf content: | @@ -253,6 +234,7 @@ write_files: #+END_SRC *** Git Configuration Script + #+BEGIN_SRC yaml - path: /home/cl/setup_git.sh owner: 'cl:cl' @@ -267,6 +249,7 @@ write_files: #+END_SRC *** Repository Setup Script + #+BEGIN_SRC yaml - path: /home/cl/setup_repos.sh owner: 'cl:cl' @@ -278,6 +261,7 @@ write_files: #+END_SRC *** User Setup Script + #+BEGIN_SRC yaml - path: /home/cl/setup_user_all.sh owner: 'cl:cl' @@ -292,6 +276,7 @@ write_files: #+END_SRC *** OpenAI Bot Blocking Script + #+BEGIN_SRC yaml - path: /home/cl/openai_block_access.sh owner: 'cl:cl' @@ -311,6 +296,7 @@ write_files: #+END_SRC *** Tmux Configuration + #+BEGIN_SRC yaml - path: /home/cl/.tmux.conf owner: 'cl:cl' @@ -547,46 +533,6 @@ runcmd: - sudo -u cl /bin/bash /home/cl/setup_user_all.sh #+END_SRC -** SSL Setup Script - -Add a script to set up SSL certificates manually when needed. - -#+BEGIN_SRC yaml -write_files: - - path: /home/cl/setup_ssl.sh - owner: cl:cl - permissions: '0755' - content: | - #!/bin/bash - # This script sets up SSL certificates using Let's Encrypt - - # Check if domain is provided - if [ $# -eq 0 ]; then - echo "Please provide a domain name as an argument." - exit 1 - fi - - DOMAIN=$1 - - # Install certbot if not already installed - if ! command -v certbot &> /dev/null; then - sudo apt update - sudo apt install -y certbot python3-certbot-nginx - fi - - # Obtain SSL certificate - sudo certbot --nginx -d $DOMAIN --non-interactive --agree-tos --email your-email@example.com --redirect - - # Setup auto-renewal - (crontab -l 2>/dev/null; echo "0 12 * * * /usr/bin/certbot renew --quiet") | crontab - - - echo "SSL certificate has been set up for $DOMAIN" - -runcmd: - # Install certbot - - apt update - - apt install -y certbot python3-certbot-nginx -#+END_SRC ** Conclusion This concludes the documentation for our ~cloudinit.yml~ file. To generate the actual YAML file from this Org document, you can use the following Emacs command: @@ -602,7 +548,12 @@ Or in an Org-mode babel shell block: This will create the ~cloudinit.yml~ file with all the code blocks in the correct order and with proper indentation. Remember to review the generated YAML file to ensure all indentations are correct, as YAML is sensitive to indentation. + + * Terraform and AWS +:PROPERTIES: +:header-args:hcl: :tangle main.tf :mkdirp yes +:END: ** Introduction This tutorial will guide you through creating a ~main.tf~ file for setting up @@ -610,6 +561,41 @@ basic AWS infrastructure using Terraform. We'll explain each resource, why it's necessary, and the order in which they should be created. The code blocks in this file can be tangled to create the final ~main.tf~ file. +#+name: tf-graph +#+begin_src powershell :results output + terraform graph +#+end_src + +#+begin_src dot :var g=tf-graph :file tf-graph.png :exports results + $g +#+end_src + +#+RESULTS: +[[file:tf-graph.png]] + +#+name: tf-plan +#+begin_src powershell :results output :exports none + terraform plan +#+end_src + +#+name: tf-destroy +#+begin_src powershell :results output :exports none + terraform destroy -auto-approve +#+end_src + +** Define Global Variables + +#+begin_src hcl :tangle variables.tf + variable "host_os" { + type = string + default = "windows" + } +#+end_src + +#+begin_src hcl :tangle terraform.tfvars + host_os = "windows" +#+end_src + ** Virtual Private Cloud (VPC) We start with creating a VPC, which is a virtual network dedicated to your AWS @@ -778,6 +764,7 @@ This EC2 instance: - Uses the cloud-init configuration we specified. ** Output + Lastly, we add an output to display the public IP of our instance. #+BEGIN_SRC hcl @@ -795,8 +782,26 @@ sets up a basic AWS infrastructure. The resources are created in a logical order, with each building upon the previous ones to create a fully functional network and compute environment in AWS. -Remember to also create the necessary ~datasource.tf~ and ~providers.tf~ files -to complete your Terraform configuration. +#+name: tf-apply +#+begin_src powershell :results output :exports none + terraform apply -auto-approve +#+end_src -To tangle this file and create ~main.tf~, you can use the following Emacs -command: ~C-c C-v t~ + +* Web App + +Write a tiny web application to test if the infrastructure is successful. + +#+begin_src lisp + (ql:quickload #:hunchentoot) + + (defpackage sample-web-app + (:use #:cl) + (:import-from #:hunchentoot + #:define-easy-handler + #:start + #:stop) + (:documentation "Sample web app package.")) + + (in-package #:sample-web-app) +#+end_src