Loading...
Loading...
Investigates, diagnoses, and mitigates Google Compute Engine virtual machine (VM) host events, including planned infrastructure maintenance (migrateOnHostMaintenance, terminateOnHostMaintenance), physical host hardware errors (hostError), guest OS terminations (guestTerminate), and Spot or Preemptible VM preemptions (compute.instances.preempted) using shutdown-script metadata. Use ONLY when an existing Compute Engine VM unexpectedly reboots, stops, crashes, is preempted, or becomes unresponsive due to physical host maintenance, hardware errors, or Local SSD recovery. DO NOT read or use for creating new standalone VMs, Managed Instance Group (MIG) autoscaler scale-out or scale-in failures, Google Kubernetes Engine (GKE) clusters, application OOM debugging, or VPC firewall issues.
npx skill4agent add google/skills compute-host-events-troubleshootingcompute.instances.hostErrorcompute.instances.migrateOnHostMaintenancecompute.instances.terminateOnHostMaintenancecompute.instances.preemptedcompute.instances.automaticRestartcompute.instances.guestTerminate_Required._AllLogsMCP Tools:,get_instance_basic_info,list_instances,list_instance_attached_disks,start_instance,stop_instance,reset_instance,delete_instanceget_zone_operation
gcloudgcloudgcloudgcloud --versionget_instance_basic_infolist_instanceslist_instance_attached_disksstart_instancestop_instancereset_instancedelete_instanceget_zone_operationroles/mcp.toolUserget_instance_basic_infoInstanceBasicInfonameidstatusmachine_typecreate_timeguest_acceleratorsschedulingresourceStatus.upcomingMaintenanceset-schedulingperform-maintenancegcloudgcloud auth logingcloud auth application-default loginGOOGLE_APPLICATION_CREDENTIALSgcloud config set project {project_id}
gcloud config set compute/region {region}
gcloud config set compute/zone {zone}us-central1us-central1-agcloud services enable compute.googleapis.com logging.googleapis.com monitoring.googleapis.comroles/compute.viewerroles/logging.viewerroles/logging.privateLogViewersystem_eventactivityroles/logging.viewAccessorroles/bigquery.jobUserroles/bigquery.dataViewerbqroles/compute.instanceAdmin.v1set-schedulingperform-maintenanceadd-metadataremove-metadataroles/logging.configWriterroles/mcp.toolUsercloudaudit.googleapis.com/system_eventcloudaudit.googleapis.com/activity--freshness=7dgcloud logging read--freshness1dhostErrorpreemptedcompute.instances.hostErrorscheduling.automaticRestarttrue--restart-on-failurecompute.instances.automaticRestartcompute.instances.hostErroroperation.first="true"operation.last="true"scheduling.localSsdRecoveryTimeoutcompute.instances.migrateOnHostMaintenancecompute.instances.terminateOnHostMaintenancesystem_eventcompute.instances.upcomingMaintenanceonHostMaintenanceMIGRATEcompute.instances.migrateOnHostMaintenanceTERMINATEcompute.instances.terminateOnHostMaintenancecompute.instances.automaticRestartautomaticRestart: truecompute.instances.preemptedgcloud logging read{vm_name}gcloud logging read 'resource.type="gce_instance" AND (log_id("cloudaudit.googleapis.com/system_event") OR log_id("cloudaudit.googleapis.com/activity")) AND protoPayload.methodName=~"(compute\.instances\.(hostError|migrateOnHostMaintenance|terminateOnHostMaintenance|preempted|automaticRestart|upcomingMaintenance|guestTerminate|repair.*)|v1\.compute\.instances\.(stop|reset|delete|performMaintenance|simulateMaintenanceEvent))" AND protoPayload.resourceName:"{vm_name}"' \
--project={project_id} \
--freshness=7d \
--limit=25 \
--format="table(timestamp, logName.basename(), protoPayload.methodName, protoPayload.authenticationInfo.principalEmail, operation.first, operation.last, protoPayload.status.message)"guestTerminatehostErrorv1.compute.instances.stopsudo shutdown -h nowpoweroffsystemctl poweroffgcloud logging read 'resource.type="gce_instance" AND log_id("cloudaudit.googleapis.com/system_event") AND protoPayload.methodName="compute.instances.guestTerminate" AND protoPayload.resourceName:"{vm_name}"' \
--project={project_id} \
--freshness=7d \
--limit=20 \
--format="yaml(timestamp, protoPayload.methodName, protoPayload.authenticationInfo.principalEmail, protoPayload.status)"guestTerminatecompute.instances.guestTerminatesystem_eventprotoPayload.authenticationInfo.principalEmailshutdownpoweroffv1.compute.instances.stopresetdeletecloudaudit.googleapis.com/activityprincipalEmailjournalctl -b -1/var/log/syslog/var/log/messages/var/log/auth.loglast -x shutdown rebootgcloud compute instances get-serial-port-output {vm_name} --project={project_id} --zone={zone} --port=1hostErrormigrateOnHostMaintenanceterminateOnHostMaintenancepreemptedautomaticRestartguestTerminatecloudaudit.googleapis.com/system_event_Defaultsystem_event_RequiredPrerequisites forSQL:_Required._AllLogs
- Upgrade the
bucket to Log Analytics (seek user approval first):_Requiredshgcloud logging buckets update _Required --project={project_id} --location=global --enable-analytics- In Log Analytics
schemas,_AllLogshas BigQuery typeresource.labels(notJSON). Always extract fields withSTRUCTandJSON_VALUE(resource.labels.zone)— grouping directly on rawJSON_VALUE(resource.labels.instance_id)expressions fails withJSON.Grouping by expressions of type JSON is not allowed
SELECT
JSON_VALUE(resource.labels.zone) AS zone,
JSON_VALUE(resource.labels.instance_id) AS instance_id,
REGEXP_EXTRACT(proto_payload.audit_log.resource_name, r'[^/]+$') AS vm_name,
proto_payload.audit_log.method_name AS event_type,
COUNT(*) AS event_count
FROM `{project_id}.global._Required._AllLogs`
WHERE
resource.type = 'gce_instance'
AND timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 30 DAY)
AND proto_payload.audit_log.method_name IN (
'compute.instances.hostError',
'compute.instances.migrateOnHostMaintenance',
'compute.instances.terminateOnHostMaintenance',
'compute.instances.preempted',
'compute.instances.automaticRestart',
'compute.instances.guestTerminate'
)
GROUP BY zone, instance_id, vm_name, event_type
ORDER BY event_count DESC;A4X MaxA4XA4A3A2G4G2-metalH4DX4Z3Z4DZ4Mgcloud# 1. Check a single instance for scheduling policy and upcoming maintenance (Compute Engine API):
gcloud compute instances describe {vm_name} --project={project_id} --zone={zone} \
--format="yaml(name, scheduling, resourceStatus.upcomingMaintenance)"
# 2. List upcoming maintenance windows across all instances in the project:
gcloud compute instances list --project={project_id} \
--format="table(name, zone.basename(), resourceStatus.upcomingMaintenance.type, resourceStatus.upcomingMaintenance.windowStartTime, resourceStatus.upcomingMaintenance.windowEndTime, resourceStatus.upcomingMaintenance.canReschedule, resourceStatus.upcomingMaintenance.maintenanceStatus)"
# 3. Query from inside the Guest OS via the Metadata Server:
curl -s "http://metadata.google.internal/computeMetadata/v1/instance/upcoming-maintenance?alt=json" \
-H "Metadata-Flavor: Google"gcloudresourceStatus.upcomingMaintenancetypeSCHEDULEDUNSCHEDULEDmaintenanceStatusPENDINGONGOINGcanRescheduletruefalsewindowStartTimewindowEndTimelatestWindowStartTime/computeMetadata/v1/instance/upcoming-maintenance?alt=json"NONE"maintenanceType"maintenanceType": "SCHEDULED"perform-maintenancecanReschedulemaintenanceStatusperform-maintenancecanRescheduletruemaintenanceStatusPENDINGcanReschedule: falsemaintenanceStatus: ONGOINGperform-maintenanceTERMINATEgcloud compute instances perform-maintenance {vm_name} --project={project_id} --zone={zone}roles/compute.instanceAdmin.v1MIGRATEN2N2DC3C3DC4N4E2MIGRATETERMINATEcompute.instances.terminateOnHostMaintenancescheduling.onHostMaintenanceTERMINATEMIGRATE--maintenance-policy=MIGRATE--restart-on-failurescheduling.automaticRestart: true--no-restart-on-failuregcloud compute instances set-schedulingonHostMaintenanceautomaticRestarthostErrorTimeoutSecondslocalSsdRecoveryTimeoutgcloud compute instances set-scheduling --helpinstances.setSchedulingTERMINATED--provisioning-model--preemptibleautomaticRestartgcloud compute instances set-scheduling {vm_name} \
--project={project_id} \
--zone={zone} \
--maintenance-policy=MIGRATE \
--restart-on-failure--maintenance-policy=MIGRATE--maintenance-policy=TERMINATE--restart-on-failureTERMINATE--no-restart-on-failureE2MIGRATEprovisioningModel: SPOTA4X MaxA4XA4A3A2G4G2N1N2*-metalH4DZ4MX4X5Z3Z4DZ4M--host-error-timeout-secondshostError--host-error-timeout-seconds90300gcloud90330instances.setSchedulinggcloud compute instances set-scheduling {vm_name} --project={project_id} --zone={zone} --host-error-timeout-seconds=90--local-ssd-recovery-timeouthostErrorcompute.instances.hostErroroperation.first="true"operation.last="true"scheduling.localSsdRecoveryTimeout0168168Z4MZ3z3-highmem-192-highlssd-metal6Z4DZ4MZ31--local-ssd-recovery-timeout=0--local-ssd-recovery-timeout168--local-ssd-recovery-timeout=24# Fast restart (0 hours wait) for stateless cache/scratch Local SSD workloads:
gcloud compute instances set-scheduling {vm_name} --project={project_id} --zone={zone} --local-ssd-recovery-timeout=0compute.instances.preemptedcompute.instances.preemptedshutdown-scripthttp://metadata.google.internal/computeMetadata/v1/instance/preemptedMetadata-Flavor: GoogleTRUEFALSEgcloud compute instances add-metadata {vm_name} \
--project={project_id} \
--zone={zone} \
--metadata=shutdown-script='#!/bin/bash
PREEMPTED=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/preempted" -H "Metadata-Flavor: Google")
if [ "$PREEMPTED" = "TRUE" ]; then
echo "Spot VM preemption detected; flushing checkpoints..."
# Insert fast (<20s), idempotent checkpoint or drain command here
fi'gcloud compute instances remove-metadata {vm_name} --project={project_id} --zone={zone} --keys=shutdown-scriptGCE VM Lifecycle Events Monitoringvm-lifecycle-eventsGCE VM Lifecycle Events Monitoringvm-lifecycle-events--config-from-filemethodSTRINGprotoPayload.methodNamegcloud logging metrics create --log-filter# Save as vm-lifecycle-events-metric.yaml:
name: vm-lifecycle-events
description: "Count of Compute Engine VM lifecycle and host events by method"
filter: >-
resource.type="gce_instance" AND
(log_id("cloudaudit.googleapis.com/activity") OR log_id("cloudaudit.googleapis.com/system_event")) AND
operation.first="true"
metricDescriptor:
metricKind: DELTA
valueType: INT64
unit: "1"
labels:
- key: method
valueType: STRING
description: "Compute Engine lifecycle or host event method"
labelExtractors:
method: >-
REGEXP_EXTRACT(protoPayload.methodName, "(recreateInstance|hostError|automaticRestart|guestTerminate|terminateOnHostMaintenance|preempted|insert|stop|delete|reset|start)")gcloud logging metrics create vm-lifecycle-events --project={project_id} --config-from-file=vm-lifecycle-events-metric.yamlgoogle_agent_for_sap.agent-for-sap-hana-availability-monitoringgoogle_agent_for_sap.agent-for-sap-netweaver-availability-monitoringgcloud compute instances describe {vm_name} --project={project_id} --zone={zone} \
--format="yaml(scheduling, resourceStatus.upcomingMaintenance)"gcloud compute instances remove-metadata {vm_name} --project={project_id} --zone={zone} --keys=shutdown-script--quietgcloud compute instances delete {vm_name} --project={project_id} --zone={zone} --delete-disks=all