spb/forge Public MIT
Forge — LLM training from scratch in pure C++20 + Metal on Apple Silicon.
C++ 61.2%
C 23%
Python 7.6%
TeX 7.2%
CMake 1.1%
1#!/usr/bin/env python323#--------------------------------------------------------------------------------------------------------------------------------------------------------------4#5# SingleHeader/MakeSingleHeader.py6#7# Copyright 2020-2024 Apple Inc.8#9# Licensed under the Apache License, Version 2.0 (the "License");10# you may not use this file except in compliance with the License.11# You may obtain a copy of the License at12#13# http://www.apache.org/licenses/LICENSE-2.014#15# Unless required by applicable law or agreed to in writing, software16# distributed under the License is distributed on an "AS IS" BASIS,17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.18# See the License for the specific language governing permissions and19# limitations under the License.20#21#--------------------------------------------------------------------------------------------------------------------------------------------------------------2223import argparse24import datetime25import logging26import os27import re28import subprocess29import sys3031#--------------------------------------------------------------------------------------------------------------------------------------------------------------3233class HeaderPrefix( object ):34 __template = ( '//\n'35 '// {file}\n'36 '//\n'37 '// {meta_data}\n'38 '//\n'39 '// Copyright 2020-2024 Apple Inc.\n'40 '//\n'41 '// Licensed under the Apache License, Version 2.0 (the "License");\n'42 '// you may not use this file except in compliance with the License.\n'43 '// You may obtain a copy of the License at\n'44 '//\n'45 '// http://www.apache.org/licenses/LICENSE-2.0\n'46 '//\n'47 '// Unless required by applicable law or agreed to in writing, software\n'48 '// distributed under the License is distributed on an "AS IS" BASIS,\n'49 '// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n'50 '// See the License for the specific language governing permissions and\n'51 '// limitations under the License.\n'52 '//\n'53 '\n' )5455 __template_commit = 'Autogenerated from commit {commit}.'56 __template_date = 'Autogenerated on %B %d, %Y.'5758 def __init__( self, file ):59 self.__file = file6061 def __str__( self ):62 return self.__template.format( file = self.__file, meta_data = self.__meta_data_string() )6364 def __get_commit_hash( self ):65 git_commit_hash = None6667 try:68 git_dir = os.path.dirname( os.path.realpath( __file__ ) )69 proc = subprocess.Popen( [ 'git', 'rev-parse', 'HEAD' ], cwd = git_dir, stdout = subprocess.PIPE, stderr = subprocess.PIPE )70 git_commit_hash = proc.stdout.read().decode( 'utf-8', 'replace' ).strip()71 except:72 logging.error( 'Failed to determine git commit hash!' )73 pass7475 return git_commit_hash7677 def __get_commit_string( self ):78 meta_data = None79 git_commit_hash = self.__get_commit_hash()8081 if git_commit_hash:82 meta_data = self.__template_commit.format( commit = git_commit_hash )8384 return meta_data8586 def __get_date_string( self ):87 today = datetime.date.today()88 89 return today.strftime( self.__template_date )9091 def __meta_data_string( self ):92 meta_data = self.__get_commit_string()9394 if not meta_data:95 meta_data = self.__get_date_string()9697 return meta_data9899#--------------------------------------------------------------------------------------------------------------------------------------------------------------100101class SingleHeader( object ):102 __pragma_once = '#pragma once\n\n'103104 def __init__( self ):105 self.__header_paths = list()106107 def __str__( self ):108 return self.process()109110 def append( self, header_path ):111 self.__header_paths.append( header_path )112113 def process( self ):114 out_header = self.__pragma_once115116 self.__included_headers = set()117 self.__base_path = list()118119 for header_path in self.__header_paths:120 out_header += self.__process_header( header_path )121122 return self.__strip_empty_lines( out_header )123124 def __read_header( self, path ):125 path = os.path.realpath( path )126127 try:128 f = open( path, 'r' )129 except:130 raise RuntimeError( 'Failed to open file \"' + path + '\" for read!' )131132 return f.read()133134 def __strip_pragma_once( self, header ):135 return re.sub( '\\s*#pragma once\s*\\/\\/-*\\n', '', header )136137 def __strip_comments( self, header ):138 return re.sub( '^//.*\\n', '', header, flags = re.MULTILINE )139140 def __strip_empty_lines( self, header ):141 return re.sub( '\\n\\n+', '\\n\\n', header, flags = re.MULTILINE )142143 def __substitute_include_directive( self, match ):144 header_path = match.group( 'HEADER_PATH' )145146 logging.info( '\tSubstituting \"' + header_path + '\"...' )147148 return self.__process_header( os.path.join( self.__base_path[-1], header_path ) )149150 def __process_include_directives( self, header ):151 return re.sub( '^\\s*#include\\s\\"(?P<HEADER_PATH>\\S*)\\"', self.__substitute_include_directive, header, flags = re.MULTILINE )152153 def __process_foundation_directives( self, header ):154 if header.find("#include <Foundation/Foundation.hpp>") != -1:155 logging.info( '\tSubstituting <Foundation/Foundation.hpp>...' )156 return header.replace("#include <Foundation/Foundation.hpp>", self.__process_header( os.path.join( self.__base_path[-1], "../Foundation/Foundation.hpp" ) ) )157 return header158159160 def __process_header( self, header_path ):161 out_header = '' 162163 header_path = os.path.realpath( header_path )164165 if not header_path in self.__included_headers:166 logging.info( 'Processing \"' + header_path + '\"...' )167168 self.__base_path.append( os.path.dirname( header_path ) )169 self.__included_headers.add( header_path )170 171 out_header = self.__read_header( header_path )172 out_header = self.__strip_pragma_once( out_header )173 out_header = self.__strip_comments( out_header )174 out_header = self.__process_include_directives( out_header )175 out_header = self.__process_foundation_directives( out_header )176177 self.__base_path.pop()178 else:179 logging.info( '\tSkipping \"' + header_path + '\"...' )180181 return out_header182183#--------------------------------------------------------------------------------------------------------------------------------------------------------------184185def create_argument_parser():186 parser = argparse.ArgumentParser()187 base_path = os.path.dirname( os.path.realpath( __file__ ) )188 output_path = os.path.join( base_path, 'Metal.hpp' )189190 parser.add_argument( '-o', '--output', dest = 'output_path', metavar = 'PATH', default = output_path, help = 'Output path for the single header file.' )191 parser.add_argument( '-v', '--verbose', action = 'store_true', help = 'Show verbose output.' )192 parser.add_argument( dest = 'header_paths', metavar = 'HEADER_FILE', nargs='+', help = 'Input header file.' )193194 return parser195196#--------------------------------------------------------------------------------------------------------------------------------------------------------------197198def parse_arguments():199 parser = create_argument_parser()200 args = parser.parse_args()201202 if args.verbose:203 logging.getLogger().setLevel( logging.INFO )204 else:205 logging.getLogger().setLevel( logging.ERROR )206207 return args208209#--------------------------------------------------------------------------------------------------------------------------------------------------------------210211def make_header( args ):212 prefix = HeaderPrefix( os.path.basename( args.output_path ) )213 header = SingleHeader()214 215 for header_path in args.header_paths:216 header.append( header_path )217218 return str( prefix ) + str( header )219220#--------------------------------------------------------------------------------------------------------------------------------------------------------------221222def make_dir( path ):223 try:224 if not os.path.exists( path ):225 os.makedirs( path )226 except os.error:227 pass228 except:229 raise230231#--------------------------------------------------------------------------------------------------------------------------------------------------------------232233def write_header( args, content ):234 path = os.path.realpath( args.output_path )235236 logging.info( 'Writing \"' + path + '\"...' )237238 make_dir( os.path.dirname( path ) )239240 try:241 f = open( path, 'w' )242 except:243 raise RuntimeError( 'Failed to open file \"' + path + '\" for write!' )244245 f.write( content )246247#--------------------------------------------------------------------------------------------------------------------------------------------------------------248249if __name__ == '__main__':250 result = -1251 252 try:253 if sys.getdefaultencoding().lower() == 'ascii':254 reload( sys )255 sys.setdefaultencoding( 'utf-8' )256257 args = parse_arguments()258 header = make_header( args )259260 write_header( args, header )261262 result = 0263264 except ( KeyboardInterrupt, SystemExit ):265 pass266 except:267 raise268269 sys.exit( result )270271#--------------------------------------------------------------------------------------------------------------------------------------------------------------272