helm repo add grafana https://grafana.github.io/helm-charts
helm repo update
helm show values grafana/promtail > promtail-values.yaml

nano promtail-values.yaml

config:
  clients:
    - url: http://12.34.56.78:3100/loki/api/v1/push  # Update the Loki API endpoint

helm install promtail grafana/promtail -f promtail-values.yaml --namespace monitoring --create-namespace

k get pods -n monitoring

kubectl logs -n monitoring -l app.kubernetes.io/name=promtail

k get pods -n monitoring

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
helm search repo prometheus-community
helm install prometheus prometheus-community/prometheus --namespace monitoring --create-namespace
k get pods -n monitoring

helm show values prometheus-community/prometheus > prom-values.yaml

nano prom-values.yaml

remoteWrite:
  - url: "http://<external-prometheus-ip>:9090/api/v1/write"

server:
  enabled: true
  persistence:
    enabled: false  # Disable persistent storage

alertmanager:
  enabled: true
  persistence:
    enabled: false  # Disable persistence to avoid using PersistentVolumes

helm upgrade --install prometheus prometheus-community/prometheus -f prom-values.yaml --namespace monitoring

k get pods -n monitoring
kubectl describe pod <prometheus-server-pod-name> -n monitoring

# cat persistent-volumes.yaml 
apiVersion: v1
kind: PersistentVolume
metadata:
  name: prometheus-server-pv
spec:
  capacity:
    storage: 10Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  local:
    path: /mnt/data/prometheus  # Path on the node verify the permissions of this dir 
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - <NODE-NAME> # Node name where PV have to be create in local storage /mnt/data/prometheus

# cat persistent-volume-claim.yaml 
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: prometheus-server
  namespace: monitoring
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: prometheus-storage  # Use the StorageClass if needed

# cat storage-class.yaml 
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: prometheus-storage
provisioner: kubernetes.io/no-provisioner  # This is for local storage (not cloud)
volumeBindingMode: WaitForFirstConsumer