SPB Git

spb/forge Public MIT

Forge — LLM training from scratch in pure C++20 + Metal on Apple Silicon.

C++ 61.2% C 23% Python 7.6% TeX 7.2% CMake 1.1%
8.8 KB · 272 lines python
Raw Blame History
1#!/usr/bin/env python323#--------------------------------------------------------------------------------------------------------------------------------------------------------------4#5# SingleHeader/MakeSingleHeader.py6#7# Copyright 2020-2024 Apple Inc.8#9# Licensed under the Apache License, Version 2.0 (the "License");10# you may not use this file except in compliance with the License.11# You may obtain a copy of the License at12#13#     http://www.apache.org/licenses/LICENSE-2.014#15# Unless required by applicable law or agreed to in writing, software16# distributed under the License is distributed on an "AS IS" BASIS,17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.18# See the License for the specific language governing permissions and19# limitations under the License.20#21#--------------------------------------------------------------------------------------------------------------------------------------------------------------2223import argparse24import datetime25import logging26import os27import re28import subprocess29import sys3031#--------------------------------------------------------------------------------------------------------------------------------------------------------------3233class HeaderPrefix( object ):34	__template 			= ( '//\n'35							'// {file}\n'36							'//\n'37							'// {meta_data}\n'38							'//\n'39							'// Copyright 2020-2024 Apple Inc.\n'40							'//\n'41							'// Licensed under the Apache License, Version 2.0 (the "License");\n'42							'// you may not use this file except in compliance with the License.\n'43							'// You may obtain a copy of the License at\n'44							'//\n'45							'//     http://www.apache.org/licenses/LICENSE-2.0\n'46							'//\n'47							'// Unless required by applicable law or agreed to in writing, software\n'48							'// distributed under the License is distributed on an "AS IS" BASIS,\n'49							'// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n'50							'// See the License for the specific language governing permissions and\n'51							'// limitations under the License.\n'52							'//\n'53							'\n' )5455	__template_commit	=	'Autogenerated from commit {commit}.'56	__template_date		=	'Autogenerated on %B %d, %Y.'5758	def __init__( self, file ):59		self.__file = file6061	def __str__( self ):62		return self.__template.format( file = self.__file, meta_data = self.__meta_data_string() )6364	def __get_commit_hash( self ):65		git_commit_hash = None6667		try:68			git_dir			= os.path.dirname( os.path.realpath( __file__ ) )69			proc 			= subprocess.Popen( [ 'git', 'rev-parse', 'HEAD' ], cwd = git_dir, stdout = subprocess.PIPE, stderr = subprocess.PIPE )70			git_commit_hash = proc.stdout.read().decode( 'utf-8', 'replace' ).strip()71		except:72			logging.error( 'Failed to determine git commit hash!' )73			pass7475		return git_commit_hash7677	def __get_commit_string( self ):78		meta_data		= None79		git_commit_hash	= self.__get_commit_hash()8081		if git_commit_hash:82			meta_data = self.__template_commit.format( commit = git_commit_hash )8384		return meta_data8586	def __get_date_string( self ):87		today = datetime.date.today()88 89		return today.strftime( self.__template_date )9091	def __meta_data_string( self ):92		meta_data = self.__get_commit_string()9394		if not meta_data:95			meta_data = self.__get_date_string()9697		return meta_data9899#--------------------------------------------------------------------------------------------------------------------------------------------------------------100101class SingleHeader( object ):102	__pragma_once = '#pragma once\n\n'103104	def __init__( self ):105		self.__header_paths = list()106107	def __str__( self ):108		return self.process()109110	def append( self, header_path ):111		self.__header_paths.append( header_path )112113	def process( self ):114		out_header 				= self.__pragma_once115116		self.__included_headers	= set()117		self.__base_path 		= list()118119		for header_path in self.__header_paths:120			out_header += self.__process_header( header_path )121122		return self.__strip_empty_lines( out_header )123124	def __read_header( self, path ):125		path = os.path.realpath( path )126127		try:128			f = open( path, 'r' )129		except:130			raise RuntimeError( 'Failed to open file \"' + path + '\" for read!' )131132		return f.read()133134	def __strip_pragma_once( self, header ):135		return re.sub( '\\s*#pragma once\s*\\/\\/-*\\n', '', header )136137	def __strip_comments( self, header ):138		return re.sub( '^//.*\\n', '', header, flags = re.MULTILINE )139140	def __strip_empty_lines( self, header ):141		return re.sub( '\\n\\n+', '\\n\\n', header, flags = re.MULTILINE )142143	def __substitute_include_directive( self, match ):144		header_path = match.group( 'HEADER_PATH' )145146		logging.info( '\tSubstituting \"' + header_path + '\"...' )147148		return self.__process_header( os.path.join( self.__base_path[-1], header_path ) )149150	def __process_include_directives( self, header ):151		return re.sub( '^\\s*#include\\s\\"(?P<HEADER_PATH>\\S*)\\"', self.__substitute_include_directive, header, flags = re.MULTILINE )152153	def __process_foundation_directives( self, header ):154		if header.find("#include <Foundation/Foundation.hpp>") != -1:155			logging.info( '\tSubstituting <Foundation/Foundation.hpp>...' )156			return header.replace("#include <Foundation/Foundation.hpp>", self.__process_header( os.path.join( self.__base_path[-1], "../Foundation/Foundation.hpp" ) ) )157		return header158159160	def __process_header( self, header_path ):161		out_header = ''		162163		header_path = os.path.realpath( header_path )164165		if not header_path in self.__included_headers:166			logging.info( 'Processing \"' + header_path + '\"...' )167168			self.__base_path.append( os.path.dirname( header_path ) )169			self.__included_headers.add( header_path )170			171			out_header = self.__read_header( header_path )172			out_header = self.__strip_pragma_once( out_header )173			out_header = self.__strip_comments( out_header )174			out_header = self.__process_include_directives( out_header )175			out_header = self.__process_foundation_directives( out_header )176177			self.__base_path.pop()178		else:179			logging.info( '\tSkipping \"' + header_path + '\"...' )180181		return out_header182183#--------------------------------------------------------------------------------------------------------------------------------------------------------------184185def create_argument_parser():186	parser 			= argparse.ArgumentParser()187	base_path 		= os.path.dirname( os.path.realpath( __file__ ) )188	output_path		= os.path.join( base_path, 'Metal.hpp' )189190	parser.add_argument( '-o', '--output',  dest = 'output_path', metavar = 'PATH', default = output_path, help = 'Output path for the single header file.' )191	parser.add_argument( '-v', '--verbose', action = 'store_true',  help = 'Show verbose output.' )192	parser.add_argument( dest = 'header_paths', metavar = 'HEADER_FILE', nargs='+', help = 'Input header file.' )193194	return parser195196#--------------------------------------------------------------------------------------------------------------------------------------------------------------197198def parse_arguments():199	parser	= create_argument_parser()200	args	= parser.parse_args()201202	if args.verbose:203		logging.getLogger().setLevel( logging.INFO )204	else:205		logging.getLogger().setLevel( logging.ERROR )206207	return args208209#--------------------------------------------------------------------------------------------------------------------------------------------------------------210211def make_header( args ):212	prefix = HeaderPrefix( os.path.basename( args.output_path ) )213	header = SingleHeader()214	215	for header_path in args.header_paths:216		header.append( header_path )217218	return str( prefix ) + str( header )219220#--------------------------------------------------------------------------------------------------------------------------------------------------------------221222def make_dir( path ):223	try:224		if not os.path.exists( path ):225			os.makedirs( path )226	except os.error:227		pass228	except:229		raise230231#--------------------------------------------------------------------------------------------------------------------------------------------------------------232233def write_header( args, content ):234	path = os.path.realpath( args.output_path )235236	logging.info( 'Writing \"' + path + '\"...' )237238	make_dir( os.path.dirname( path ) )239240	try:241		f = open( path, 'w' )242	except:243		raise RuntimeError( 'Failed to open file \"' + path + '\" for write!' )244245	f.write( content )246247#--------------------------------------------------------------------------------------------------------------------------------------------------------------248249if __name__ == '__main__':250	result = -1251	252	try:253		if sys.getdefaultencoding().lower() == 'ascii':254			reload( sys )255			sys.setdefaultencoding( 'utf-8' )256257		args 	= parse_arguments()258		header 	= make_header( args )259260		write_header( args, header )261262		result = 0263264	except ( KeyboardInterrupt, SystemExit ):265	 	pass266	except:267	 	raise268269	sys.exit( result )270271#--------------------------------------------------------------------------------------------------------------------------------------------------------------272