2021-11-15T09:21:29.542014685Z level=info ts=2021-11-15T09:21:29.540Z caller=main.go:216 msg="Starting Alertmanager" version="(version=0.21.0, branch=rhaos-4.7-rhel-8, revision=7d7727749b9e72d483091a58e1a13cb7d4f4fa62)"
2021-11-15T09:21:29.542014685Z level=info ts=2021-11-15T09:21:29.540Z caller=main.go:217 build_context="(go=go1.15.7, user=root@9e3ad46b3963, date=20210609-08:49:37)"
2021-11-15T09:21:29.688729257Z level=warn ts=2021-11-15T09:21:29.685Z caller=cluster.go:228 component=cluster msg="failed to join cluster" err="3 errors occurred:\n\t* Failed to resolve alertmanager-main-0.alertmanager-operated:9094: lookup alertmanager-main-0.alertmanager-operated on 172.30.0.10:53: no such host\n\t* Failed to resolve alertmanager-main-1.alertmanager-operated:9094: lookup alertmanager-main-1.alertmanager-operated on 172.30.0.10:53: no such host\n\t* Failed to resolve alertmanager-main-2.alertmanager-operated:9094: lookup alertmanager-main-2.alertmanager-operated on 172.30.0.10:53: no such host\n\n"
2021-11-15T09:21:29.688729257Z level=info ts=2021-11-15T09:21:29.685Z caller=cluster.go:230 component=cluster msg="will retry joining cluster every 10s"
2021-11-15T09:21:29.688729257Z level=warn ts=2021-11-15T09:21:29.685Z caller=main.go:307 msg="unable to join gossip mesh" err="3 errors occurred:\n\t* Failed to resolve alertmanager-main-0.alertmanager-operated:9094: lookup alertmanager-main-0.alertmanager-operated on 172.30.0.10:53: no such host\n\t* Failed to resolve alertmanager-main-1.alertmanager-operated:9094: lookup alertmanager-main-1.alertmanager-operated on 172.30.0.10:53: no such host\n\t* Failed to resolve alertmanager-main-2.alertmanager-operated:9094: lookup alertmanager-main-2.alertmanager-operated on 172.30.0.10:53: no such host\n\n"
2021-11-15T09:21:29.688729257Z level=info ts=2021-11-15T09:21:29.687Z caller=cluster.go:623 component=cluster msg="Waiting for gossip to settle..." interval=2s
2021-11-15T09:21:29.840749198Z level=info ts=2021-11-15T09:21:29.840Z caller=coordinator.go:119 component=configuration msg="Loading configuration file" file=/etc/alertmanager/config/alertmanager.yaml
2021-11-15T09:21:29.841639033Z level=info ts=2021-11-15T09:21:29.840Z caller=coordinator.go:131 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/config/alertmanager.yaml
2021-11-15T09:21:29.844588876Z level=info ts=2021-11-15T09:21:29.844Z caller=main.go:485 msg=Listening address=127.0.0.1:9093
2021-11-15T09:21:31.688010032Z level=info ts=2021-11-15T09:21:31.687Z caller=cluster.go:648 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000160539s
2021-11-15T09:21:36.069883883Z level=info ts=2021-11-15T09:21:36.065Z caller=coordinator.go:119 component=configuration msg="Loading configuration file" file=/etc/alertmanager/config/alertmanager.yaml
2021-11-15T09:21:36.069883883Z level=info ts=2021-11-15T09:21:36.065Z caller=coordinator.go:131 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/config/alertmanager.yaml
2021-11-15T09:21:39.697216107Z level=info ts=2021-11-15T09:21:39.697Z caller=cluster.go:640 component=cluster msg="gossip settled; proceeding" elapsed=10.009414061s
2021-11-15T09:21:44.715090814Z level=warn ts=2021-11-15T09:21:44.715Z caller=cluster.go:438 component=cluster msg=refresh result=failure addr=alertmanager-main-2.alertmanager-operated:9094 err="1 error occurred:\n\t* Failed to resolve alertmanager-main-2.alertmanager-operated:9094: lookup alertmanager-main-2.alertmanager-operated on 172.30.0.10:53: no such host\n\n"
2021-11-15T09:21:59.695862111Z level=warn ts=2021-11-15T09:21:59.695Z caller=cluster.go:438 component=cluster msg=refresh result=failure addr=alertmanager-main-2.alertmanager-operated:9094 err="1 error occurred:\n\t* Failed to resolve alertmanager-main-2.alertmanager-operated:9094: lookup alertmanager-main-2.alertmanager-operated on 172.30.0.10:53: no such host\n\n"
2021-11-16T08:34:38.427677160Z level=info ts=2021-11-16T08:34:38.423Z caller=coordinator.go:119 component=configuration msg="Loading configuration file" file=/etc/alertmanager/config/alertmanager.yaml
2021-11-16T08:34:38.427677160Z level=info ts=2021-11-16T08:34:38.423Z caller=coordinator.go:131 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/config/alertmanager.yaml
2021-11-16T08:36:49.115902451Z level=warn ts=2021-11-16T08:36:49.115Z caller=notify.go:674 component=dispatcher receiver=Default integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://this-endpoint.does/not-exist\": dial tcp 200.160.2.95:443: connect: connection timed out"
2021-11-16T08:36:49.116291870Z level=warn ts=2021-11-16T08:36:49.115Z caller=notify.go:674 component=dispatcher receiver=Default integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://this-endpoint.does/not-exist\": dial tcp 200.160.2.95:443: connect: connection timed out"
2021-11-16T08:39:38.427419866Z level=error ts=2021-11-16T08:39:38.427Z caller=dispatch.go:309 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Default/webhook[0]: notify retry canceled after 4 attempts: Post \"https://this-endpoint.does/not-exist\": context deadline exceeded"
2021-11-16T08:39:38.428637761Z level=error ts=2021-11-16T08:39:38.428Z caller=dispatch.go:309 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Default/webhook[0]: notify retry canceled after 4 attempts: Post \"https://this-endpoint.does/not-exist\": context deadline exceeded"
2021-11-16T08:41:48.124399641Z level=warn ts=2021-11-16T08:41:48.123Z caller=notify.go:674 component=dispatcher receiver=Default integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://this-endpoint.does/not-exist\": dial tcp 200.160.2.95:443: connect: connection timed out"
2021-11-16T08:44:38.429245827Z level=error ts=2021-11-16T08:44:38.428Z caller=dispatch.go:309 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Default/webhook[0]: notify retry canceled after 4 attempts: Post \"https://this-endpoint.does/not-exist\": context deadline exceeded"
2021-11-16T08:44:44.252018413Z level=warn ts=2021-11-16T08:44:44.251Z caller=notify.go:674 component=dispatcher receiver=Default integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://this-endpoint.does/not-exist\": dial tcp 200.160.2.95:443: connect: connection timed out"
2021-11-16T08:44:44.252018413Z level=warn ts=2021-11-16T08:44:44.251Z caller=notify.go:674 component=dispatcher receiver=Critical integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://this-endpoint.does/not-exist\": dial tcp 200.160.2.95:443: connect: connection timed out"
2021-11-16T08:46:47.131868802Z level=warn ts=2021-11-16T08:46:47.131Z caller=notify.go:674 component=dispatcher receiver=Default integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://this-endpoint.does/not-exist\": dial tcp 200.160.2.95:443: connect: connection timed out"
2021-11-16T08:47:34.490605785Z level=warn ts=2021-11-16T08:47:34.490Z caller=notify.go:674 component=dispatcher receiver=Default integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=3 err="Post \"https://this-endpoint.does/not-exist\": dial tcp 200.160.2.95:443: i/o timeout"
2021-11-16T08:47:34.490869829Z level=error ts=2021-11-16T08:47:34.490Z caller=dispatch.go:309 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Default/webhook[0]: notify retry canceled after 5 attempts: Post \"https://this-endpoint.does/not-exist\": context deadline exceeded"
2021-11-16T08:47:34.495474536Z level=error ts=2021-11-16T08:47:34.495Z caller=dispatch.go:309 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Critical/webhook[0]: notify retry canceled after 4 attempts: Post \"https://this-endpoint.does/not-exist\": context deadline exceeded"
