Update infrastructure
This commit is contained in:
parent
338c30ea95
commit
63ffea201d
1 changed files with 141 additions and 136 deletions
|
|
@ -1,123 +1,99 @@
|
|||
#+TITLE: Creating AWS Infrastructure for Common Lisp Web-Application using cloudinit && terraform
|
||||
#+PROPERTY: header-args:hcl :tangle main.tf :mkdirp yes
|
||||
#+title: IaC AWS for ml-survey
|
||||
#+author: Marcus Kammer
|
||||
|
||||
#+begin_src python :noweb "yes"
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
import yaml
|
||||
* About
|
||||
|
||||
def represent_ordereddict(dumper, data):
|
||||
return dumper.represent_mapping('tag:yaml.org,2002:map', data.items())
|
||||
This org file is part of the [[https://code.siemens.com/marcus.kammer/ml-survey][ml-survey]] repository. It is meant to be the
|
||||
infrastructure-as-code documentation.
|
||||
|
||||
yaml.add_representer(OrderedDict, represent_ordereddict)
|
||||
|
||||
# Define the order of keys
|
||||
key_order = [
|
||||
"local",
|
||||
"keyboard",
|
||||
"timezone",
|
||||
"groups",
|
||||
"users",
|
||||
"packages",
|
||||
"package_update",
|
||||
"package_upgrade"
|
||||
]
|
||||
|
||||
# Initialize OrderedDict with the defined keys
|
||||
cloudinit_data = OrderedDict((key, None) for key in key_order)
|
||||
|
||||
<<local-settings>>
|
||||
<<timezone-setting>>
|
||||
<<groups-creation>>
|
||||
<<user-creation-configuration>>
|
||||
<<package-install>>
|
||||
|
||||
# Remove any keys with None values
|
||||
cloudinit_data = OrderedDict((k, v) for k, v in cloudinit_data.items() if v is not None)
|
||||
|
||||
with open("cloudinit1.yaml", 'w') as file:
|
||||
file.write("#cloud-config\n")
|
||||
yaml.dump(cloudinit_data, file, default_flow_style=False)
|
||||
#+end_src
|
||||
|
||||
#+RESULTS:
|
||||
: None
|
||||
|
||||
* Cloudinit
|
||||
:PROPERTIES:
|
||||
:header-args:yaml: :tangle cloudinit.yml
|
||||
:END:
|
||||
** Introduction
|
||||
|
||||
This document explains the structure and content of our ~cloudinit.yml~ file,
|
||||
which is used to initialize our AWS EC2 instance. The code blocks in this file
|
||||
can be tangled to create the final ~cloudinit.yml~ file.
|
||||
|
||||
** Locale and Keyboard Settings
|
||||
** Cloud-Config Header
|
||||
Every cloud-init file should start with "#cloud-config". This tells cloud-init that the file is a cloud-config file.
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
#cloud-config
|
||||
#+END_SRC
|
||||
|
||||
** Locale and Keyboard Settings
|
||||
Set the system locale and keyboard layout.
|
||||
|
||||
#+name: local-settings
|
||||
#+begin_src python
|
||||
cloudinit_data["local"] = "en_US.UTF-8"
|
||||
cloudinit_data["keyboard"] = {"layout": "en_US.UTF-8"}
|
||||
#+end_src
|
||||
#+BEGIN_SRC yaml
|
||||
locale: en_US.UTF-8
|
||||
keyboard:
|
||||
layout: us
|
||||
#+END_SRC
|
||||
|
||||
** Timezone Setting
|
||||
|
||||
Set the system timezone.
|
||||
|
||||
#+name: timezone-setting
|
||||
#+begin_src python
|
||||
cloudinit_data["timezone"] = "Europe/Berlin"
|
||||
#+end_src
|
||||
|
||||
** Groups Creation
|
||||
#+BEGIN_SRC yaml
|
||||
timezone: Europe/Berlin
|
||||
#+END_SRC
|
||||
|
||||
** Group Creation
|
||||
Create any necessary system groups.
|
||||
|
||||
#+name: groups-creation
|
||||
#+begin_src python
|
||||
cloudinit_data["groups"] = ["nginxgroup"]
|
||||
#+end_src
|
||||
#+BEGIN_SRC yaml
|
||||
groups:
|
||||
- nginxgroup
|
||||
#+END_SRC
|
||||
|
||||
** User Creation and Configuration
|
||||
Create and configure users. Here we're creating two users: a system user for Nginx and a regular user for administration.
|
||||
|
||||
Create and configure users. Here we're creating two users: a system user for
|
||||
Nginx and a regular user for administration.
|
||||
|
||||
#+name: user-creation-configuration
|
||||
#+begin_src python
|
||||
def read_ssh_key(identity="id_ed25519"):
|
||||
home_dir = os.path.expanduser("~")
|
||||
ssh_file_path = os.path.join(home_dir, ".ssh", f"{identity}.pub")
|
||||
|
||||
try:
|
||||
with open(ssh_file_path, "r") as ssh_file:
|
||||
return ssh_file.readline().strip()
|
||||
except FileNotFoundError:
|
||||
print(f"SSH key file not found: {ssh_file_path}")
|
||||
return None
|
||||
|
||||
cloudinit_data["users"] = [{"name": "nginxuser",
|
||||
"system": "true",
|
||||
"shell": "/usr/sbin/nologin",
|
||||
"groups": ["nginxgroup"],
|
||||
"sudo": "null"},
|
||||
{"name": "cl",
|
||||
"groups": ["users", "admin"],
|
||||
"sudo": "ALL=(ALL) NOPASSWD:ALL",
|
||||
"shell": "/bin/bash",
|
||||
"ssh_authorized_keys": [read_ssh_key()]}]
|
||||
#+end_src
|
||||
#+BEGIN_SRC yaml
|
||||
users:
|
||||
- name: nginxuser
|
||||
system: true
|
||||
shell: /usr/sbin/nologin
|
||||
groups: nginxgroup
|
||||
sudo: null
|
||||
- name: cl
|
||||
groups: users, admin
|
||||
sudo: ALL=(ALL) NOPASSWD:ALL
|
||||
shell: /bin/bash
|
||||
ssh_authorized_keys:
|
||||
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIA+46Y3AHPLJgz8KK61doqH3jBX2TL3TJvZsJrB9Km03 visua@xps-8930
|
||||
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMIHJ5qpMIKL7N3nC0GG1O4ygtkqOlQuZReoik6xGBxn marcus@XPS-13-9380.local
|
||||
- ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB6xSH5nE0uy0C0kglpp4EqrbbW2CrBeAIj+X6Sf2pd0 XPS-8930-Ubuntu_22
|
||||
#+END_SRC
|
||||
|
||||
** Package Installation
|
||||
|
||||
Install necessary packages.
|
||||
|
||||
#+name: package-install
|
||||
#+begin_src python :results none
|
||||
cloudinit_data["packages"] = ["detachtty", "fail2ban", "ufw", "unattended-upgrades", "sbcl", "mosh", "tmux", "git", "nginx", "certbot", "python3-certbot-nginx", "build-essential", "libzstd-dev", "libsqlite3-dev", "sqlite3", "curl", "wget"]
|
||||
cloudinit_data["package_update"] = True
|
||||
cloudinit_data["package_upgrade"] = True
|
||||
#+end_src
|
||||
#+BEGIN_SRC yaml
|
||||
packages:
|
||||
- detachtty
|
||||
- fail2ban
|
||||
- ufw
|
||||
- unattended-upgrades
|
||||
- sbcl
|
||||
- mosh
|
||||
- tmux
|
||||
- git
|
||||
- nginx
|
||||
- certbot
|
||||
- python3-certbot-nginx
|
||||
- build-essential
|
||||
- libzstd-dev
|
||||
- libsqlite3-dev
|
||||
- sqlite3
|
||||
- curl
|
||||
- wget
|
||||
|
||||
package_update: true
|
||||
package_upgrade: true
|
||||
#+END_SRC
|
||||
|
||||
** File Writing
|
||||
|
||||
|
|
@ -128,6 +104,7 @@ write_files:
|
|||
#+END_SRC
|
||||
|
||||
*** Automatic Upgrades Configuration
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
- path: /etc/apt/apt.conf.d/20auto-upgrades
|
||||
content: |
|
||||
|
|
@ -138,6 +115,7 @@ write_files:
|
|||
#+END_SRC
|
||||
|
||||
*** SSH Configuration
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
- path: /etc/ssh/sshd_config
|
||||
content: |
|
||||
|
|
@ -163,6 +141,7 @@ write_files:
|
|||
#+END_SRC
|
||||
|
||||
*** Fail2Ban Configuration
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
- path: /etc/fail2ban/jail.local
|
||||
content: |
|
||||
|
|
@ -195,6 +174,7 @@ write_files:
|
|||
#+END_SRC
|
||||
|
||||
*** Nginx Configuration
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
- path: /etc/nginx/nginx.conf
|
||||
content: |
|
||||
|
|
@ -224,6 +204,7 @@ write_files:
|
|||
#+END_SRC
|
||||
|
||||
*** Nginx Reverse Proxy Configuration
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
- path: /etc/nginx/sites-available/reverse-proxy.conf
|
||||
content: |
|
||||
|
|
@ -253,6 +234,7 @@ write_files:
|
|||
#+END_SRC
|
||||
|
||||
*** Git Configuration Script
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
- path: /home/cl/setup_git.sh
|
||||
owner: 'cl:cl'
|
||||
|
|
@ -267,6 +249,7 @@ write_files:
|
|||
#+END_SRC
|
||||
|
||||
*** Repository Setup Script
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
- path: /home/cl/setup_repos.sh
|
||||
owner: 'cl:cl'
|
||||
|
|
@ -278,6 +261,7 @@ write_files:
|
|||
#+END_SRC
|
||||
|
||||
*** User Setup Script
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
- path: /home/cl/setup_user_all.sh
|
||||
owner: 'cl:cl'
|
||||
|
|
@ -292,6 +276,7 @@ write_files:
|
|||
#+END_SRC
|
||||
|
||||
*** OpenAI Bot Blocking Script
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
- path: /home/cl/openai_block_access.sh
|
||||
owner: 'cl:cl'
|
||||
|
|
@ -311,6 +296,7 @@ write_files:
|
|||
#+END_SRC
|
||||
|
||||
*** Tmux Configuration
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
- path: /home/cl/.tmux.conf
|
||||
owner: 'cl:cl'
|
||||
|
|
@ -547,46 +533,6 @@ runcmd:
|
|||
- sudo -u cl /bin/bash /home/cl/setup_user_all.sh
|
||||
#+END_SRC
|
||||
|
||||
** SSL Setup Script
|
||||
|
||||
Add a script to set up SSL certificates manually when needed.
|
||||
|
||||
#+BEGIN_SRC yaml
|
||||
write_files:
|
||||
- path: /home/cl/setup_ssl.sh
|
||||
owner: cl:cl
|
||||
permissions: '0755'
|
||||
content: |
|
||||
#!/bin/bash
|
||||
# This script sets up SSL certificates using Let's Encrypt
|
||||
|
||||
# Check if domain is provided
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "Please provide a domain name as an argument."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DOMAIN=$1
|
||||
|
||||
# Install certbot if not already installed
|
||||
if ! command -v certbot &> /dev/null; then
|
||||
sudo apt update
|
||||
sudo apt install -y certbot python3-certbot-nginx
|
||||
fi
|
||||
|
||||
# Obtain SSL certificate
|
||||
sudo certbot --nginx -d $DOMAIN --non-interactive --agree-tos --email your-email@example.com --redirect
|
||||
|
||||
# Setup auto-renewal
|
||||
(crontab -l 2>/dev/null; echo "0 12 * * * /usr/bin/certbot renew --quiet") | crontab -
|
||||
|
||||
echo "SSL certificate has been set up for $DOMAIN"
|
||||
|
||||
runcmd:
|
||||
# Install certbot
|
||||
- apt update
|
||||
- apt install -y certbot python3-certbot-nginx
|
||||
#+END_SRC
|
||||
|
||||
** Conclusion
|
||||
This concludes the documentation for our ~cloudinit.yml~ file. To generate the actual YAML file from this Org document, you can use the following Emacs command:
|
||||
|
|
@ -602,7 +548,12 @@ Or in an Org-mode babel shell block:
|
|||
This will create the ~cloudinit.yml~ file with all the code blocks in the correct order and with proper indentation.
|
||||
|
||||
Remember to review the generated YAML file to ensure all indentations are correct, as YAML is sensitive to indentation.
|
||||
|
||||
|
||||
* Terraform and AWS
|
||||
:PROPERTIES:
|
||||
:header-args:hcl: :tangle main.tf :mkdirp yes
|
||||
:END:
|
||||
** Introduction
|
||||
|
||||
This tutorial will guide you through creating a ~main.tf~ file for setting up
|
||||
|
|
@ -610,6 +561,41 @@ basic AWS infrastructure using Terraform. We'll explain each resource, why it's
|
|||
necessary, and the order in which they should be created. The code blocks in
|
||||
this file can be tangled to create the final ~main.tf~ file.
|
||||
|
||||
#+name: tf-graph
|
||||
#+begin_src powershell :results output
|
||||
terraform graph
|
||||
#+end_src
|
||||
|
||||
#+begin_src dot :var g=tf-graph :file tf-graph.png :exports results
|
||||
$g
|
||||
#+end_src
|
||||
|
||||
#+RESULTS:
|
||||
[[file:tf-graph.png]]
|
||||
|
||||
#+name: tf-plan
|
||||
#+begin_src powershell :results output :exports none
|
||||
terraform plan
|
||||
#+end_src
|
||||
|
||||
#+name: tf-destroy
|
||||
#+begin_src powershell :results output :exports none
|
||||
terraform destroy -auto-approve
|
||||
#+end_src
|
||||
|
||||
** Define Global Variables
|
||||
|
||||
#+begin_src hcl :tangle variables.tf
|
||||
variable "host_os" {
|
||||
type = string
|
||||
default = "windows"
|
||||
}
|
||||
#+end_src
|
||||
|
||||
#+begin_src hcl :tangle terraform.tfvars
|
||||
host_os = "windows"
|
||||
#+end_src
|
||||
|
||||
** Virtual Private Cloud (VPC)
|
||||
|
||||
We start with creating a VPC, which is a virtual network dedicated to your AWS
|
||||
|
|
@ -778,6 +764,7 @@ This EC2 instance:
|
|||
- Uses the cloud-init configuration we specified.
|
||||
|
||||
** Output
|
||||
|
||||
Lastly, we add an output to display the public IP of our instance.
|
||||
|
||||
#+BEGIN_SRC hcl
|
||||
|
|
@ -795,8 +782,26 @@ sets up a basic AWS infrastructure. The resources are created in a logical
|
|||
order, with each building upon the previous ones to create a fully functional
|
||||
network and compute environment in AWS.
|
||||
|
||||
Remember to also create the necessary ~datasource.tf~ and ~providers.tf~ files
|
||||
to complete your Terraform configuration.
|
||||
#+name: tf-apply
|
||||
#+begin_src powershell :results output :exports none
|
||||
terraform apply -auto-approve
|
||||
#+end_src
|
||||
|
||||
To tangle this file and create ~main.tf~, you can use the following Emacs
|
||||
command: ~C-c C-v t~
|
||||
|
||||
* Web App
|
||||
|
||||
Write a tiny web application to test if the infrastructure is successful.
|
||||
|
||||
#+begin_src lisp
|
||||
(ql:quickload #:hunchentoot)
|
||||
|
||||
(defpackage sample-web-app
|
||||
(:use #:cl)
|
||||
(:import-from #:hunchentoot
|
||||
#:define-easy-handler
|
||||
#:start
|
||||
#:stop)
|
||||
(:documentation "Sample web app package."))
|
||||
|
||||
(in-package #:sample-web-app)
|
||||
#+end_src
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue