blob: 589937caa716aa8cc594e48d3e443217672c1c87 [file] [log] [blame]
#!/usr/bin/env python3
# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# This script computes the number of concurrent jobs that can run in the
# build as a function of the machine. It accepts a set of key value pairs
# given by repeated --memory-per-job arguments. For example:
# $ --memory-per-job dart=1GB
# The result is a json map printed to stdout that gives the number of
# concurrent jobs allowed of each kind. For example:
# {"dart": 8}
# Some memory can be held out of the calculation with the --reserve-memory flag.
import argparse
import ctypes
import json
import multiprocessing
import os
import re
import subprocess
import sys
UNITS = {'B': 1, 'KB': 2**10, 'MB': 2**20, 'GB': 2**30, 'TB': 2**40}
# See
# and
class MEMORYSTATUSEX(ctypes.Structure):
_fields_ = [
("dwLength", ctypes.c_ulong),
("dwMemoryLoad", ctypes.c_ulong),
("ullTotalPhys", ctypes.c_ulonglong),
("ullAvailPhys", ctypes.c_ulonglong),
("ullTotalPageFile", ctypes.c_ulonglong),
("ullAvailPageFile", ctypes.c_ulonglong),
("ullTotalVirtual", ctypes.c_ulonglong),
("ullAvailVirtual", ctypes.c_ulonglong),
("sullAvailExtendedVirtual", ctypes.c_ulonglong),
def GetTotalMemory():
if sys.platform in ('win32', 'cygwin'):
stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
success = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
return stat.ullTotalPhys if success else 0
elif sys.platform.startswith('linux'):
if os.path.exists("/proc/meminfo"):
with open("/proc/meminfo") as meminfo:
memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
for line in meminfo:
match = memtotal_re.match(line)
if match:
return float( * 2**10
elif sys.platform == 'darwin':
return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
except Exception:
return 0
return 0
def ParseSize(string):
i = next(i for (i, c) in enumerate(string) if not c.isdigit())
number = string[:i].strip()
unit = string[i:].strip()
return int(float(number) * UNITS[unit])
class ParseSizeAction(argparse.Action):
def __call__(self, parser, args, values, option_string=None):
sizes = getattr(args, self.dest, [])
for value in values:
(k, v) = value.split('=', 1)
sizes.append((k, ParseSize(v)))
setattr(args, self.dest, sizes)
def Main():
parser = argparse.ArgumentParser()
help='Key value pairings (dart=1GB) giving an estimate of the amount of '
'memory needed for the class of job.')
help='The amount of memory to be held out of the amount for jobs to use.')
args = parser.parse_args()
total_memory = GetTotalMemory()
# Ensure the total memory used in the calculation below is at least 0
mem_total_bytes = max(0, total_memory - args.reserve_memory)
# Ensure the number of cpus used in the calculation below is at least 1
cpu_cap = multiprocessing.cpu_count()
cpu_cap = 1
concurrent_jobs = {}
for job, memory_per_job in args.memory_per_job:
# Calculate the number of jobs that will fit in memory. Ensure the
# value is at least 1.
num_concurrent_jobs = int(max(1, mem_total_bytes / memory_per_job))
# Cap the number of jobs by the number of cpus available.
concurrent_jobs[job] = min(num_concurrent_jobs, cpu_cap)
return 0
if __name__ == '__main__':