diff --git a/infrastructure.org b/infrastructure.org new file mode 100644 index 0000000..b2d225c --- /dev/null +++ b/infrastructure.org @@ -0,0 +1,500 @@ +#+title: IaC AWS for ml-survey +#+author: Marcus Kammer +#+email: marcus.kammer@mailbox.org + +* About + +This org file is part of the [[https://code.metalisp.dev/marcuskammer/dev.metalisp.survey][ml-survey]] repository. It is meant to be the +infrastructure-as-code documentation. + +* Cloudinit +:PROPERTIES: +:header-args:yaml: :tangle cloudinit.yml +:END: +** Introduction + +This document explains the structure and content of our ~cloudinit.yml~ file, +which is used to initialize our AWS EC2 instance. The code blocks in this file +can be tangled to create the final ~cloudinit.yml~ file. + +** Cloud-Config Header +Every cloud-init file should start with "#cloud-config". This tells cloud-init that the file is a cloud-config file. + +#+BEGIN_SRC yaml +#cloud-config +#+END_SRC + +** Locale and Keyboard Settings +Set the system locale and keyboard layout. + +#+BEGIN_SRC yaml +locale: en_US.UTF-8 +keyboard: + layout: us +#+END_SRC + +** Timezone Setting +Set the system timezone. + +#+BEGIN_SRC yaml +timezone: Europe/Berlin +#+END_SRC + +** Group Creation +Create any necessary system groups. + +#+BEGIN_SRC yaml +groups: + - nginxgroup +#+END_SRC + +** User Creation and Configuration +Create and configure users. Here we're creating two users: a system user for Nginx and a regular user for administration. + +#+BEGIN_SRC yaml +users: + - name: cl + groups: users, admin + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + ssh_authorized_keys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB6xSH5nE0uy0C0kglpp4EqrbbW2CrBeAIj+X6Sf2pd0 XPS-8930-Ubuntu_22 +#+END_SRC + +** Package Installation +Install necessary packages. + +#+BEGIN_SRC yaml +packages: + - detachtty + - fail2ban + - ufw + - unattended-upgrades + - sbcl + - mosh + - tmux + - git + - nginx + - certbot + - python3-certbot-nginx + - build-essential + - libzstd-dev + - libsqlite3-dev + - sqlite3 + - curl + - wget + +package_update: true +package_upgrade: true +#+END_SRC + +** File Writing + +Write configuration files and scripts to the instance. + +#+BEGIN_SRC yaml +write_files: + - path: /etc/apt/apt.conf.d/20auto-upgrades + content: | + APT::Periodic::Update-Package-Lists "1"; + APT::Periodic::Download-Upgradeable-Packages "1"; + APT::Periodic::AutocleanInterval "7"; + APT::Periodic::Unattended-Upgrade "1"; + - path: /etc/ssh/sshd_config + content: | + Include /etc/ssh/sshd_config.d/*.conf + MaxAuthTries 3 + AuthorizedKeysFile .ssh/authorized_keys + PasswordAuthentication no + AuthenticationMethods publickey + PubkeyAuthentication yes + PermitRootLogin no + KbdInteractiveAuthentication no + UsePAM yes + AllowAgentForwarding no + AllowTcpForwarding yes + X11Forwarding no + PrintMotd no + KexAlgorithms curve25519-sha256@libssh.org + Ciphers chacha20-poly1305@openssh.com + MACs hmac-sha2-512-etm@openssh.com + AcceptEnv LANG LC_* + Subsystem sftp /usr/lib/openssh/sftp-server + AllowUsers cl + - path: /etc/fail2ban/jail.local + content: | + [DEFAULT] + bantime = 3600 + findtime = 600 + maxretry = 3 + banaction = ufw + + [sshd] + enabled = true + port = 22 + logpath = /var/log/auth.log + + [sshd-ddos] + filter = sshd + enabled = true + port = ssh + logpath = /var/log/auth.log + maxretry = 5 + bantime = 600 +#+END_SRC + +** Run Commands + +Execute commands after the instance has been set up. + +#+BEGIN_SRC yaml +runcmd: + - ufw default deny incoming + - ufw default allow outgoing + - ufw allow 22/tcp + - ufw allow mosh + - ufw enable + - systemctl enable fail2ban && systemctl start fail2ban + - systemctl restart sshd +#+END_SRC + +** Conclusion +This concludes the documentation for our ~cloudinit.yml~ file. To generate the actual YAML file from this Org document, you can use the following Emacs command: + +~C-c C-v t~ + +Or in an Org-mode babel shell block: + +#+BEGIN_SRC emacs-lisp :results silent +(org-babel-tangle) +#+END_SRC + +This will create the ~cloudinit.yml~ file with all the code blocks in the correct order and with proper indentation. + +Remember to review the generated YAML file to ensure all indentations are correct, as YAML is sensitive to indentation. + +* Terraform and AWS +:PROPERTIES: +:header-args:hcl: :tangle main.tf :mkdirp yes +:END: +** Introduction + +This tutorial will guide you through creating a ~main.tf~ file for setting up +basic AWS infrastructure using Terraform. We'll explain each resource, why it's +necessary, and the order in which they should be created. The code blocks in +this file can be tangled to create the final ~main.tf~ file. + +#+name: tf-graph +#+begin_src powershell :results output + terraform graph +#+end_src + +#+begin_src dot :var g=tf-graph :file tf-graph.png :exports results + $g +#+end_src + +#+RESULTS: +[[file:tf-graph.png]] + +#+name: tf-plan +#+begin_src powershell :results output :exports none + terraform plan +#+end_src + +#+name: tf-destroy +#+begin_src powershell :results output :exports none + terraform destroy -auto-approve +#+end_src + +#+begin_src hcl :tangle providers.tf +# This file configures the providers used in this Terraform configuration. +# Providers are plugins that Terraform uses to manage resources. + +terraform { + # Specify the required providers and their versions + required_providers { + # AWS provider for managing AWS resources + aws = { + source = "hashicorp/aws" # Source of the provider + version = "~> 4.16" # Version constraint for the provider + } + # Cloud-init provider for generating cloud-init configs + cloudinit = { + source = "hashicorp/cloudinit" + version = "~> 2.2.0" + } + } + # Specify the required version of Terraform itself + required_version = ">= 1.2.0" +} + +# Configure the AWS Provider +provider "aws" { + region = "eu-central-1" # Specify the AWS region to use + shared_credentials_files = ["~/.aws/credentials"] # Path to the AWS credentials file +} + +# Configure the cloudinit Provider +# No specific configuration needed for cloudinit, but declaring it here makes it available for use +provider "cloudinit" {} +#+end_src + +** Define Global Variables + +#+begin_src hcl :tangle variables.tf + variable "host_os" { + type = string + default = "windows" + } +#+end_src + +#+begin_src hcl :tangle terraform.tfvars + host_os = "windows" +#+end_src + +** Virtual Private Cloud (VPC) + +We start with creating a VPC, which is a virtual network dedicated to your AWS +account. It's the foundation for all other resources. + +#+BEGIN_SRC hcl :tangle main.tf +resource "aws_vpc" "mlsurvey_vpc" { + cidr_block = "10.0.0.0/16" + enable_dns_hostnames = true + enable_dns_support = true + tags = { + Name = "ml-survey-vpc" + } +} +#+END_SRC + +This VPC: +- Has a CIDR block of 10.0.0.0/16, allowing for up to 65,536 IP addresses. +- Enables DNS hostnames and support, which are necessary for EC2 instances to have DNS names. + +** Subnet + +Next, we create a subnet within our VPC. Subnets allow you to partition your +network to group resources together. + +#+BEGIN_SRC hcl :tangle main.tf +resource "aws_subnet" "mlsurvey_public_subnet" { + vpc_id = aws_vpc.mlsurvey_vpc.id + cidr_block = "10.0.1.0/24" + map_public_ip_on_launch = true + availability_zone = "eu-central-1a" + tags = { + Name = "ml-survey-public" + } +} +#+END_SRC + +This subnet: +- Is associated with our VPC. +- Has a CIDR block of 10.0.1.0/24, allowing for up to 256 IP addresses. +- Automatically assigns public IP addresses to instances launched in it. +- Is located in the eu-central-1a availability zone. + +** Internet Gateway + +An Internet Gateway allows communication between your VPC and the internet. + +#+BEGIN_SRC hcl :tangle main.tf +resource "aws_internet_gateway" "mlsurvey_internet_gateway" { + vpc_id = aws_vpc.mlsurvey_vpc.id + tags = { + Name = "ml-survey-igw" + } +} +#+END_SRC + +This Internet Gateway is attached to our VPC, enabling internet access for +resources within the VPC. + +** Route Table + +A route table contains a set of rules (routes) that determine where network +traffic is directed. + +#+BEGIN_SRC hcl :tangle main.tf +resource "aws_route_table" "mlsurvey_public_rt" { + vpc_id = aws_vpc.mlsurvey_vpc.id + tags = { + Name = "ml-survey-rt" + } +} +#+END_SRC + +This route table is associated with our VPC and will contain the rules for +routing traffic. + +** Route + +We add a route to our route table to direct internet-bound traffic to the +Internet Gateway. + +#+BEGIN_SRC hcl :tangle main.tf +resource "aws_route" "mlsurvey_default_route" { + route_table_id = aws_route_table.mlsurvey_public_rt.id + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.mlsurvey_internet_gateway.id +} +#+END_SRC + +This route sends all traffic (0.0.0.0/0) to the Internet Gateway, allowing resources in our VPC to access the internet. + +** Route Table Association + +We associate our route table with the subnet to apply the routing rules. + +#+BEGIN_SRC hcl :tangle main.tf +resource "aws_route_table_association" "mlsurvey_public_assoc" { + subnet_id = aws_subnet.mlsurvey_public_subnet.id + route_table_id = aws_route_table.mlsurvey_public_rt.id +} +#+END_SRC + +This association ensures that the routing rules apply to resources in our subnet. + +** Security Group + +A security group acts as a virtual firewall for your instance to control inbound and outbound traffic. + +#+BEGIN_SRC hcl :tangle main.tf +resource "aws_security_group" "mlsurvey_sg" { + name = "ml-survey-sg" + description = "ml-survey security group" + vpc_id = aws_vpc.mlsurvey_vpc.id + + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} +#+END_SRC + +This security group allows all inbound and outbound traffic. In a production environment, you would typically restrict this for better security. + +** Key Pair + +A key pair is used to securely SSH into your EC2 instances. + +#+BEGIN_SRC hcl :tangle main.tf +resource "aws_key_pair" "mlsurvey_auth" { + key_name = "ml-survey-key" + public_key = file("~/.ssh/ml-survey-key.pub") +} +#+END_SRC + +This resource uploads your public key to AWS, allowing you to use the +corresponding private key to SSH into instances. + +** EC2 Instance + +This file defines data sources, which fetch and compute information for use in +other parts of your Terraform configuration. + +This data source processes the cloud-init configuration file (cloudinit.yml) +for use with EC2 instances. + +#+begin_src hcl :tangle datasource.tf +data "cloudinit_config" "config" { + gzip = true # Compress the cloud-init data to save space + base64_encode = true # Encode the data in base64 for proper transmission to EC2 + + part { + content_type = "text/cloud-config" # Specify that this is a cloud-config file + content = file("${path.module}/cloudinit.yml") # Read the content of the cloudinit.yml file + } +} +#+end_src + +Amazon Machine Image (AMI) Lookup + +This data source finds the latest Ubuntu 22.04 LTS AMI for use with EC2 +instances. + +#+begin_src hcl :tangle datasource.tf +data "aws_ami" "server_ami" { + most_recent = true # Get the most recent version of the AMI + owners = ["099720109477"] # This ID represents Canonical, the company behind Ubuntu + + # Filter to find Ubuntu 22.04 LTS (Jammy Jellyfish) AMIs + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"] + } + + # Ensure we're getting a hardware virtual machine (HVM) AMI + filter { + name = "virtualization-type" + values = ["hvm"] + } + + # Specify that we want an x86_64 architecture (suitable for t2.micro instances) + filter { + name = "architecture" + values = ["x86_64"] + } +} +#+end_src + +Finally, we create an EC2 instance, which is a virtual server in Amazon's +Elastic Compute Cloud (EC2) for running applications. + +#+BEGIN_SRC hcl :tangle main.tf +resource "aws_instance" "dev_node" { + instance_type = "t2.micro" + ami = data.aws_ami.server_ami.id + key_name = aws_key_pair.mlsurvey_auth.id + vpc_security_group_ids = [aws_security_group.mlsurvey_sg.id] + subnet_id = aws_subnet.mlsurvey_public_subnet.id + user_data = data.cloudinit_config.config.rendered + user_data_replace_on_change = true + tags = { + Name = "dev-node" + } +} +#+END_SRC + +This EC2 instance: +- Uses the t2.micro instance type. +- Uses the AMI specified in the ~aws_ami~ data source. +- Uses the key pair we created for SSH access. +- Is placed in our VPC and subnet. +- Has the security group we created applied to it. +- Uses the cloud-init configuration we specified. + +** Output + +Lastly, we add an output to display the public IP of our instance. + +#+BEGIN_SRC hcl :tangle main.tf + output "dev_node_public_ip" { + value = aws_instance.dev_node.public_ip + } +#+END_SRC + +This output will be displayed after Terraform applies the configuration, making +it easy to find the IP address of your new instance. + +** Conclusion + +By tangling all these code blocks, you'll have a complete ~main.tf~ file that +sets up a basic AWS infrastructure. The resources are created in a logical +order, with each building upon the previous ones to create a fully functional +network and compute environment in AWS. + +#+name: tf-apply +#+begin_src powershell :results output :exports none + terraform apply -auto-approve +#+end_src diff --git a/terraform.tfvars b/terraform.tfvars new file mode 100644 index 0000000..cca0879 --- /dev/null +++ b/terraform.tfvars @@ -0,0 +1 @@ +host_os = "windows" diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..ddbff88 --- /dev/null +++ b/variables.tf @@ -0,0 +1,4 @@ +variable "host_os" { + type = string + default = "windows" +}