Loading...
Loading...
Diagnoses, mitigates, and prevents Google Compute Engine resource availability and capacity stockout errors (ZONE_RESOURCE_POOL_EXHAUSTED, ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS, and machine type or accelerator unavailable in zone). Use ONLY when VM creation, Managed Instance Group (MIG) scale-out, or stopped instance startup fails due to regional or zonal hardware capacity exhaustion for GPUs, TPUs, or specific CPU shapes. DO NOT read or use for IAM permission errors, project quota limit increases without stockouts, general greenfield VM creation, or unexpected host reboots on already-running VMs.
npx skill4agent add google/skills compute-resource-availability-troubleshootingMCP Tools:,list_machine_types,list_accelerator_types,get_zone_operation,list_instance_group_managers,list_managed_instances,list_reservationsget_reservation_details
gcloudgcloudgcloudgcloud --versionroles/mcp.toolUsergcloudlist_machine_typeslist_accelerator_typesget_zone_operationerror.errorsZONE_RESOURCE_POOL_EXHAUSTEDZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILSQUOTA_EXCEEDEDlist_instance_group_managerslist_managed_instancescurrentActionlastAttempt.errorslist_instance_group_managerslist_managed_instancesget_instance_group_manager_basic_infozonegcloud--region={region}list_reservationsget_reservation_detailsspecificReservation.countspecificReservation.inUseCountgcloudcreate_instanceprovisioning_modelFLEX_STARTSPOTmax_run_durationinstance_termination_actionreservation_affinitygcloud logging readgcloud beta compute advice capacitygcloud compute instances bulk creategcloud compute instance-groups managed updategcloud compute reservations createdeletegcloudgcloud auth logingcloud auth application-default loginGOOGLE_APPLICATION_CREDENTIALSgcloud config set project {project_id}
gcloud config set compute/region {region}
gcloud config set compute/zone {zone}
gcloud services enable compute.googleapis.com logging.googleapis.come2-micro{project_id}{region}{zone}us-central1us-central1-aroles/compute.viewerroles/logging.viewer_Default._AllLogsroles/logging.viewAccessorroles/bigquery.jobUserroles/bigquery.dataViewerroles/compute.instanceAdmin.v1roles/iam.serviceAccountUserroles/compute.admincompute.reservations.createcompute.reservations.updatecompute.reservations.deleteroles/compute.instanceAdmin.v1roles/mcp.toolUserZONE_RESOURCE_POOL_EXHAUSTEDZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILSQUOTA_EXCEEDEDZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILSerrorInfo.reasonconfiguration_availabilityerrorInfo.metadata.zonesAvailableg2-standard-4c4-standard-32us-central1-aus-central1-bus-central1-cus-central1-fc4-standard-16TERMINATEDRUNNINGgcloud logging read--freshnessgcloud logging read1d--freshness=7d--freshness=14dprotoPayload.response.error.errors[0].code.reasonZONE_RESOURCE_POOL_EXHAUSTEDZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILSQUOTA_EXCEEDEDgcloud logging read \
'resource.type="gce_instance" AND log_id("cloudaudit.googleapis.com/activity") AND (protoPayload.status.message:("ZONE_RESOURCE_POOL_EXHAUSTED" OR "does not have enough resources available" OR "resource pool exhausted" OR "does not exist in zone" OR "QUOTA_EXCEEDED" OR "Quota") OR protoPayload.response.error.errors.code:("ZONE_RESOURCE_POOL_EXHAUSTED" OR "ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS" OR "QUOTA_EXCEEDED"))' \
--project={project_id} --freshness=7d --limit=20 \
--format="table(timestamp, resource.labels.zone, protoPayload.resourceName, protoPayload.response.error.errors[0].code, protoPayload.status.message)"codeQUOTA_EXCEEDEDgcloud compute regions describegcloud compute regions describe {region} --project={project_id} \
--format="table(quotas:format='table(metric,limit,usage)')"list_managed_instancesgcloud# Zonal MIG (--zone={zone}) vs. Regional MIG (--region={region}):
gcloud compute instance-groups managed list-errors {mig_name} \
--project={project_id} --zone={zone}
gcloud compute instance-groups managed list-errors {mig_name} \
--project={project_id} --region={region}cloudaudit.googleapis.com/activityERRORseverity_number >= 500_Default{project_id}.global._Default._AllLogs_Required._AllLogsERROR_Defaultgcloud logging buckets update _Default \
--project={project_id} --location=global --enable-analytics_AllLogsresource.labelsproto_payload.audit_log.requestJSONresource.labels.zoneGROUP BYSTRINGJSON_VALUE(...)SELECT
JSON_VALUE(resource.labels.zone) AS zone,
REGEXP_EXTRACT(JSON_VALUE(proto_payload.audit_log.request.machineType), r'([^/]+)$') AS machine_type,
proto_payload.audit_log.status.message AS status_message,
COUNT(*) AS stockout_count,
MIN(timestamp) AS first_seen,
MAX(timestamp) AS last_seen
FROM `{project_id}.global._Default._AllLogs`
WHERE
timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 14 DAY)
AND log_id = 'cloudaudit.googleapis.com/activity'
AND resource.type = 'gce_instance'
AND (
proto_payload.audit_log.status.message LIKE '%ZONE_RESOURCE_POOL_EXHAUSTED%'
OR proto_payload.audit_log.status.message LIKE '%does not have enough resources available%'
OR TO_JSON_STRING(proto_payload.audit_log.response.error.errors) LIKE '%ZONE_RESOURCE_POOL_EXHAUSTED%'
)
GROUP BY zone, machine_type, status_message
ORDER BY stockout_count DESC;ZONE_RESOURCE_POOL_EXHAUSTEDMachine type with name '...' does not exist in zone '...'{region}list_machine_typeslist_accelerator_typesgcloud# List all zones in {region} that offer {machine_type}:
gcloud compute machine-types list --project={project_id} \
--filter="name={machine_type} AND zone ~ ^{region}-" \
--format="table(name, zone, guestCpus, memoryMb)"
# For GPU workloads (especially N1 with attached GPUs), verify accelerator zones:
gcloud compute accelerator-types list --project={project_id} \
--filter="name={accelerator_type} AND zone ~ ^{region}-" \
--format="table(name, zone, maximumCardsPerInstance)"bulk creategcloud compute instances bulk create--region={region}--target-distribution-shape=ANY_SINGLE_ZONE--count=1--target-distribution-shape=BALANCED{region}# On instances bulk create, --target-distribution-shape is UPPERCASE (ANY, ANY_SINGLE_ZONE, BALANCED):
gcloud compute instances bulk create \
--project={project_id} --region={region} \
--name-pattern="{vm_name}-#" --count={vm_count} \
--target-distribution-shape=ANY_SINGLE_ZONE \
--machine-type={machine_type}advice capacitygcloud beta compute advice capacityestimatedUptime{region}gcloud beta compute advice capacity-historygcloud beta compute advice capacity--provisioning-model=SPOTSTANDARD--machine-typeadvice capacity--instance-selection-machine-types={machine_type_1},{machine_type_2}--instance-selection="name={selection_name},machine-type={machine_type}"advice capacity--target-distribution-shapeANYANY_SINGLE_ZONEBALANCED--provisioning-model=SPOT--instance-selection-machine-types--target-distribution-shape=ANYANY_SINGLE_ZONEBALANCEDgcloud beta compute advice capacity \
--project={project_id} --region={region} \
--provisioning-model=SPOT --size={vm_count} \
--instance-selection-machine-types={machine_type_1},{machine_type_2} \
--target-distribution-shape=ANY*-standard-64*-standard-96*-standard-128a4-highgpu-8ga3-highgpu-8ga2-highgpu-1gg2-standard-4ZONE_RESOURCE_POOL_EXHAUSTEDG2nvidia-l4G4A2nvidia-tesla-a100A3A4--acceleratorN1--reservation-affinity=none--max-run-duration10m7d--instance-termination-action=DELETESTOP--maintenance-policy=TERMINATE--request-valid-for-duration90s2h0s--default-action-on-vm-failure=do-nothing--target-distribution-shape=any-single-zone--instance-redistribution-type=none# Option A: Standalone VM with DWS Flex-Start (queues up to 2h via --request-valid-for-duration):
gcloud compute instances create {vm_name} \
--project={project_id} --zone={zone} \
--machine-type=g2-standard-4 \
--provisioning-model=FLEX_START \
--reservation-affinity=none \
--request-valid-for-duration=2h \
--max-run-duration=7d \
--instance-termination-action=DELETE \
--maintenance-policy=TERMINATE
# Option B: Instance Template + Zonal or Regional MIG Resize Request (longer queueing):
gcloud compute instance-templates create {template_name} \
--project={project_id} --machine-type=g2-standard-4 \
--provisioning-model=FLEX_START \
--reservation-affinity=none \
--max-run-duration=7d \
--instance-termination-action=DELETE \
--maintenance-policy=TERMINATE
# Queue capacity on a MIG configured per bullet 2 (--zone={zone} or --region={region}):
gcloud compute instance-groups managed resize-requests create {mig_name} \
--project={project_id} --region={region} \
--resize-request={request_name} --resize-by={vm_count}balancedanybalancedBALANCEDanyANYany-single-zoneANY_SINGLE_ZONE--instance-redistribution-typeevenEVENproactivePROACTIVEbalancedBALANCEDanyANYany-single-zoneANY_SINGLE_ZONE--instance-redistribution-type=noneproactiveevenproactivebalancedanyany-single-zoneinstances bulk creategcloud compute instance-groups managed update--target-distribution-shapeanyany-single-zonebalancedeven--instance-redistribution-typenoneproactive# Allow spillover to healthy zones without proactively deleting running VMs:
gcloud compute instance-groups managed update {mig_name} \
--project={project_id} --region={region} \
--instance-redistribution-type=none --target-distribution-shape=balanced
# Or maximize obtainability in any zone (requires --instance-redistribution-type=none):
gcloud compute instance-groups managed update {mig_name} \
--project={project_id} --region={region} \
--instance-redistribution-type=none --target-distribution-shape=anyinstanceFlexibilityPolicyc3-standard-8n2-standard-8n2d-standard-8--target-distribution-shapebalancedanyany-single-zoneeven--instance-redistribution-type=nonegcloud beta compute instance-groups managed update--instance-selection--remove-instance-selectionsinstanceFlexibilityPolicy.instanceSelectionsn4-standard-8n2-standard-8--instance-flexibility-policy--flags-file--instance-selection--force-update-on-repairgcloud beta compute instance-groups managed update {mig_name} \
--project={project_id} --region={region} \
--target-distribution-shape=balanced \
--instance-redistribution-type=none \
--instance-selection="name=primary-tier,machine-type=c3-standard-8,rank=1" \
--instance-selection="name=fallback-tier,machine-type=n2-standard-8,rank=2"roles/compute.admincompute.reservations.*{zone}{zone}--accelerator--local-ssdgcloud compute reservations create {reservation_name} \
--project={project_id} --zone={zone} \
--vm-count={vm_count} --machine-type={machine_type}anybalanced--instance-redistribution-type=noneevenproactive# Option A: Converge opportunistically on resizes without deleting VMs:
gcloud compute instance-groups managed update {mig_name} \
--project={project_id} --region={region} \
--target-distribution-shape=balanced --instance-redistribution-type=none
# Option B: Stateless strict even + proactive (rebalance manually first if >1 VM diff):
gcloud compute instance-groups managed update {mig_name} \
--project={project_id} --region={region} \
--target-distribution-shape=even --instance-redistribution-type=none
gcloud compute instance-groups managed update {mig_name} \
--project={project_id} --region={region} \
--instance-redistribution-type=proactivegcloud beta compute instance-groups managed update {mig_name} \
--project={project_id} --region={region} \
--remove-instance-selections=fallback-tier--quietgcloud compute reservations delete {reservation_name} \
--project={project_id} --zone={zone}
gcloud compute instances delete {vm_name} \
--project={project_id} --zone={zone}