Simplify: Use Coolify for Elasticsearch instead of AWS

- Remove OpenSearch/Fargate Elasticsearch from Terraform
- Elasticsearch deployed manually on Coolify (existing infrastructure)
- AWS only manages: SQS + 2 Lambdas
- Lambdas connect to Coolify Elasticsearch via URL
- Much simpler and cheaper: ~-5/month AWS only
- No VPC complexity needed for Lambdas

Author: Alexanderamiri

.github/workflows/deploy.yaml

@@ -62,11 +62,8 @@ jobs:
         working-directory: terraform
         run: terraform plan -input=false
         env:
-          TF_VAR_vpc_id: ${{ secrets.VPC_ID }}
-          TF_VAR_es_subnet_ids: ${{ secrets.ES_SUBNET_IDS }}
-          TF_VAR_assign_public_ip: false
-          TF_VAR_allowed_cidr_blocks: ${{ secrets.VPC_CIDR_BLOCKS }}
           TF_VAR_elasticsearch_password: ${{ secrets.ELASTICSEARCH_PASSWORD }}
+          TF_VAR_elasticsearch_url: ${{ secrets.ELASTICSEARCH_URL }}
           TF_VAR_webhook_secret: ${{ secrets.WEBHOOK_SECRET }}
           TF_VAR_moresleep_url: ${{ secrets.MORESLEEP_URL }}
           TF_VAR_moresleep_username: ${{ secrets.MORESLEEP_USERNAME }}
@@ -76,15 +73,8 @@ jobs:
         working-directory: terraform
         run: terraform apply -auto-approve -input=false
         env:
-          TF_VAR_vpc_id: ${{ secrets.VPC_ID }}
-          TF_VAR_es_subnet_ids: ${{ secrets.ES_SUBNET_IDS }}
-          TF_VAR_assign_public_ip: false
-          TF_VAR_allowed_cidr_blocks: ${{ secrets.VPC_CIDR_BLOCKS }}
           TF_VAR_elasticsearch_password: ${{ secrets.ELASTICSEARCH_PASSWORD }}
-          TF_VAR_elasticsearch_url: http://elasticsearch.javazone.internal:9200
-          TF_VAR_task_cpu: 1024
-          TF_VAR_task_memory: 2048
-          TF_VAR_heap_size: 1024
+          TF_VAR_elasticsearch_url: ${{ secrets.ELASTICSEARCH_URL }}
           TF_VAR_webhook_secret: ${{ secrets.WEBHOOK_SECRET }}
           TF_VAR_moresleep_url: ${{ secrets.MORESLEEP_URL }}
           TF_VAR_moresleep_username: ${{ secrets.MORESLEEP_USERNAME }}
@@ -94,32 +84,16 @@ jobs:
         working-directory: terraform
         run: |
           echo "================================================"
-          echo "✅ Deployment Complete!"
+          echo "✅ AWS Infrastructure Deployed!"
           echo "================================================"
           echo ""
           echo "🔗 Webhook URL for moresleep:"
           terraform output -raw webhook_url
           echo ""
           echo ""
-          echo "📊 OpenSearch:"
-          terraform output -raw opensearch_endpoint
-          echo ""
-          echo ""
           echo "📦 SQS Queue:"
           terraform output -raw sqs_queue_url
           echo ""
+          echo ""
+          echo "📊 Elasticsearch: Deploy on Coolify (see README)"
           echo "================================================"
-
-      - name: Wait for OpenSearch to be ready
-        run: |
-          echo "Waiting 5 minutes for OpenSearch domain to be active..."
-          sleep 300
-
-      - name: Create OpenSearch Index
-        working-directory: terraform
-        run: |
-          OS_URL=$(terraform output -raw opensearch_endpoint)
-          curl -X PUT "$OS_URL/javazone_talks" \
-            -u elastic:${{ secrets.ELASTICSEARCH_PASSWORD }} \
-            -H "Content-Type: application/json" \
-            -d @../config/index-mapping.json || echo "Index may already exist"

README.md

@@ -1,156 +1,87 @@
-# Elasticsearch for JavaZone
+# JavaZone Elasticsearch Integration
 
-Elasticsearch deployment on AWS Fargate with EFS persistent storage.
+Complete Elasticsearch integration for JavaZone - all in one repository!
 
-## Architecture
+## 🎯 What This Deploys
 
-- **Single Fargate task** running Elasticsearch 8.11
-- **EFS volume** for persistent data storage
-- **Service Discovery** for stable DNS: `elasticsearch.javazone.internal:9200`
-- **1 vCPU, 2GB RAM** (sufficient for ~10K talks)
+**AWS (via Terraform):**
+- SQS queues (main + DLQ)
+- webhook-receiver Lambda (API Gateway)
+- es-indexer-worker Lambda (SQS triggered)
 
-## Deployment
+**Coolify (manual):**
+- Elasticsearch 8.11.0
 
-### 1. Deploy Infrastructure
+**Cost: ~$2-5/month AWS + $0 Coolify = ~$2-5/month total**
 
-```bash
-cd terraform
-
-cp terraform.tfvars.example terraform.tfvars
-# Edit with your values
+---
 
-terraform init
-terraform apply
-```
+## 🚀 Quick Start
 
-**What this creates:**
-- ECS Cluster and Fargate task
-- EFS file system for data persistence
-- Security groups
-- Service discovery (optional DNS name)
-- SSM parameter for password
+### 1. Deploy Elasticsearch on Coolify
 
-**Cost:** ~$35-45/month (Fargate + EFS)
+1. Coolify → New Resource → Docker Image
+2. Image: `elasticsearch:8.11.0`
+3. Environment:
+   ```
+   discovery.type=single-node
+   xpack.security.enabled=true
+   ELASTIC_PASSWORD=bi75Xtl3KPXI4CS7QRU8TrFRpF3mV1qX
+   ES_JAVA_OPTS=-Xms1g -Xmx1g
+   ```
+4. Volume: `/data/elasticsearch` → `/usr/share/elasticsearch/data`
+5. Port: `9200`
+6. Deploy
 
-### 2. Wait for Elasticsearch to Start
+7. Create index:
+   ```bash
+   curl -X PUT "http://your-es-url:9200/javazone_talks" \
+     -u elastic:bi75Xtl3KPXI4CS7QRU8TrFRpF3mV1qX \
+     -H "Content-Type: application/json" \
+     -d @config/index-mapping.json
+   ```
 
-```bash
-# Check task status
-aws ecs list-tasks --cluster elasticsearch-javazone --region eu-central-1
-
-# Get task IP (if not using service discovery)
-aws ecs describe-tasks \
-  --cluster elasticsearch-javazone \
-  --tasks <task-arn> \
-  --region eu-central-1 | grep "privateIpv4Address"
-```
+8. Update GitHub secret with your ES URL:
+   ```bash
+   gh secret set ELASTICSEARCH_URL --body "http://your-elasticsearch-domain:9200"
+   ```
 
-### 3. Create Index
+---
 
-If using service discovery:
-```bash
-ES_URL="http://elasticsearch.javazone.internal:9200"
-```
+### 2. Deploy AWS Infrastructure (GitHub Actions)
 
-Otherwise:
-```bash
-ES_URL="http://<task-private-ip>:9200"
-```
+Push to `main` or manually trigger workflow:
 
-Create the index:
-```bash
-curl -X PUT "$ES_URL/javazone_talks" \
-  -u elastic:<your-password> \
-  -H "Content-Type: application/json" \
-  -d @../config/index-mapping.json
-```
+https://github.com/javaBin/elasticsearch-javazone/actions
 
-Verify:
-```bash
-curl -u elastic:<password> "$ES_URL/javazone_talks"
-curl -u elastic:<password> "$ES_URL/_cluster/health"
-```
+Deploys: SQS + 2 Lambdas (~2 minutes)
 
-## Access from Other Services
+---
 
-### With Service Discovery (Recommended)
-```
-ELASTICSEARCH_URL=http://elasticsearch.javazone.internal:9200
-```
+### 3. Configure Moresleep
 
-Use this in:
-- es-indexer-worker terraform.tfvars
-- libum configuration
+Use webhook URL from GitHub Actions output:
 
-### Without Service Discovery
-Use the task's private IP (changes on restart):
-```
-ELASTICSEARCH_URL=http://10.0.x.x:9200
+```properties
+WEBHOOK_ENABLED=true
+WEBHOOK_ENDPOINT=<url-from-output>
+WEBHOOK_SECRET=7faa5e7879e189dfdeab497f647a88e15abcd7be4c2209f318465e764547d258
 ```
 
-## Monitoring
+Redeploy moresleep.
 
-View task logs in AWS ECS Console:
-- ECS → Clusters → elasticsearch-javazone → Tasks → View logs tab
+---
 
-Check cluster health:
-```bash
-curl -u elastic:<password> "$ES_URL/_cluster/health"
-```
-
-Check document count:
-```bash
-curl -u elastic:<password> "$ES_URL/javazone_talks/_count"
-```
-
-## Data Persistence
-
-Data is stored on EFS and persists across task restarts. To completely reset:
+## ✅ Testing
 
+Create a talk → Check ES within 10 seconds:
 ```bash
-# Stop service
-aws ecs update-service \
-  --cluster elasticsearch-javazone \
-  --service elasticsearch-javazone \
-  --desired-count 0
-
-# Delete EFS data (requires EC2 instance or Fargate task with EFS mounted)
-# Then restart service
-aws ecs update-service \
-  --cluster elasticsearch-javazone \
-  --service elasticsearch-javazone \
-  --desired-count 1
+curl -u elastic:pass "http://your-es-url:9200/javazone_talks/_search?q=test"
 ```
 
-## Cost Estimate
-
-- **Fargate**: ~$25-30/month (1 vCPU, 2GB, always running)
-- **EFS**: ~$10-15/month (5-10GB data)
-- **SSM**: Free
-- **Service Discovery**: Free
-
-**Total: ~$35-45/month**
-
-## Scaling
-
-For higher load (unlikely needed):
-- Increase `task_cpu` and `task_memory` in terraform.tfvars
-- Apply and restart
-
-For production HA:
-- Deploy multiple tasks
-- Use Application Load Balancer
-- Multiple AZ deployment
-
-But for 10K talks, single task is sufficient.
-
-## Backup
-
-EFS data is persistent, but consider:
-- EFS automatic backups (AWS Backup)
-- Elasticsearch snapshot repository
-- Periodic manual snapshots
+---
 
-## Sample Queries
+**Total setup time: ~10 minutes**
+**Monthly cost: ~$2-5**
 
-See `config/sample-queries.sh` for common Elasticsearch queries.
+Perfect! 🎉

terraform/main.tf

@@ -213,89 +213,23 @@ resource "aws_lambda_event_source_mapping" "sqs_trigger" {
 }
 
 ################################################################################
-# OpenSearch Domain
+# Elasticsearch on Coolify
+################################################################################
+#
+# Elasticsearch is deployed on Coolify (external to this Terraform)
+#
+# Setup in Coolify:
+# 1. Deploy elasticsearch:8.11.0 container
+# 2. Set environment variables:
+#    - discovery.type=single-node
+#    - xpack.security.enabled=true
+#    - ELASTIC_PASSWORD=<use var.elasticsearch_password>
+#    - ES_JAVA_OPTS=-Xms1g -Xmx1g
+# 3. Expose port 9200
+# 4. Add persistent volume for /usr/share/elasticsearch/data
+# 5. Note the URL (e.g., http://elasticsearch.your-domain.com:9200)
+# 6. Create index using config/index-mapping.json
+#
+# Cost: Depends on your Coolify hosting (~$0-10/month)
 ################################################################################
-
-# Security Group for OpenSearch
-resource "aws_security_group" "opensearch" {
-  name        = "opensearch-javazone"
-  description = "OpenSearch for JavaZone"
-  vpc_id      = var.vpc_id
-
-  ingress {
-    from_port   = 443
-    to_port     = 443
-    protocol    = "tcp"
-    cidr_blocks = var.allowed_cidr_blocks
-    description = "HTTPS for OpenSearch"
-  }
-
-  egress {
-    from_port   = 0
-    to_port     = 0
-    protocol    = "-1"
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-}
-
-# OpenSearch Domain
-resource "aws_opensearch_domain" "javazone" {
-  domain_name    = "javazone-talks"
-  engine_version = "OpenSearch_2.11"
-
-  cluster_config {
-    instance_type  = "t3.small.search"  # ~$0.036/hour = ~$26/month
-    instance_count = 1
-    zone_awareness_enabled = false
-  }
-
-  ebs_options {
-    ebs_enabled = true
-    volume_size = 10  # GB - plenty for 10K talks
-    volume_type = "gp3"
-  }
-
-  vpc_options {
-    subnet_ids         = [var.es_subnet_ids[0]]  # Single AZ for cost savings
-    security_group_ids = [aws_security_group.opensearch.id]
-  }
-
-  advanced_security_options {
-    enabled                        = true
-    internal_user_database_enabled = true
-    master_user_options {
-      master_user_name     = var.elasticsearch_username
-      master_user_password = var.elasticsearch_password
-    }
-  }
-
-  encrypt_at_rest {
-    enabled = true
-  }
-
-  node_to_node_encryption {
-    enabled = true
-  }
-
-  domain_endpoint_options {
-    enforce_https       = true
-    tls_security_policy = "Policy-Min-TLS-1-2-2019-07"
-  }
-
-  access_policies = jsonencode({
-    Version = "2012-10-17"
-    Statement = [{
-      Effect = "Allow"
-      Principal = {
-        AWS = "*"
-      }
-      Action   = "es:*"
-      Resource = "arn:aws:es:${var.aws_region}:*:domain/javazone-talks/*"
-    }]
-  })
-
-  tags = {
-    Name = "javazone-talks"
-  }
-}