ϦiHSSKrSSKrSSKrSSKrSSKrSSKJr SSKJr SSK J r SSK J r SSK Jr \RR!\RR#\55r\RR'\S5/rSr\R,"5S 5rS r"S S \5rS rSr"SS\5r"SS\5rg)N)Path)_build)get_cache_manager) GPUTarget) GPUDriverincludec^^ ^ ^ ^SSKnUR5S:wagSSKm SSKJnJnJnJm Jm Jn "U U 4SjST R5nT RX5"U5U"U5U"U55nT RS5RnUT /Ul X8lSmT RTS-5n U UU4S jn U"U"U 5U 5(a%[ R""T R%U 55$g! g=f) NrLinux)c_charc_intc_size_tc_void_pc_char_pPOINTERc*>\rSrSrSY4SY4/rSrg)8_find_already_mmapped_dylib_on_linux..DlPhdrInfo dlpi_addr dlpi_nameN)__name__ __module__ __qualname____firstlineno___fields___static_attributes__)rrsY/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/triton/backends/amd/driver.py DlPhdrInfors ( # ( # rz libc.so.6ic >URRn[[R"U55nTUR ;a&TR X#[T[U555 gg)Nr r) contentsrrosfsdecodenamememmoveminlen)infosizedatarpctypeslib_namemax_path_lengths rcallback6_find_already_mmapped_dylib_on_linux..callback2sPMM++ Y' ( qvv  NN4CY,P Qr)platformsystemr-r r r rrr Structure CFUNCTYPECDLLdl_iterate_phdrargtypesrestypecreate_string_bufferr#r$ string_at)r.r2r r r rr callback_tr7pathr0rrr-r/s` @@@@r$_find_already_mmapped_dylib_on_linuxr>sG# KK  V%% !!%)OQXY_Q`aJ ++k2BB!+H5O#O  & &': ;Dz(+T22{{6++D122 +s 2C44C8cSn[R"S5nU(aMURU5(a&[RR U5(aU$[ SUSU35e[ U5nU(a7[RR U5(aU$[ SUSU35e/nSSKnUR5nUR5nUR(aU/U-nUH^n[RRUSS U5n[RR U5(aUs $URU5 M` [R"S 5nU(apURS 5H[n [RRX5n [RR U 5(aU s $URU 5 M] [R"S S /5R!5n U R#5V s/sH;oR%5RU5(dM(U R5SPM= n n U H<n[RR U5(aUs $URU5 M> [RRSU5n[RR U5(aU$URU5 [ SUSU35es sn f)Nzlibamdhip64.soTRITON_LIBHIP_PATHzTRITON_LIBHIP_PATH 'z' does not point to a valid zmemory mapped 'z'' in process does not point to a valid rtorchlibLD_LIBRARY_PATH:z/sbin/ldconfigz-pz/opt/rocm/lib/zcannot locate z after attempted paths )r#getenvendswithr=exists RuntimeErrorr>sitegetsitepackagesgetusersitepackagesENABLE_USER_SITEjoinappendsplit subprocess check_outputdecode splitlinesstrip)r.env_libhip_path mmapped_pathpathsrJ site_packages user_siter=env_ld_library_pathdflibslinelocsloccommon_install_paths r_get_path_to_hip_runtime_dylibrc@spHii 45O  # #H - -"''..2Q2Q" "1/1BB^_g^hijj8AL 77>>, ' ' _\N:abjaklmm E((*M((*I " m3 ww||D'5(; 77>>$  K T ))$56$**3/A Q)Aww~~a   LLO 0  " "$4d#; < C C ED*.): ^):jjl>S>ST\>] DJJL ):D ^ 77>>#  J S '',,'7B ww~~)**"" LL$% z1HP QQ _s 'L L c [R"URS55R5n[ U5nUR US35nUc[ R"5n[RRUS5n[US5nURU5 SSS5 [XU/[/5n[US5nURUR!5US3SS9nSSS5 SSS5 SSKn U R$R'X5n U R$R)U 5n U R*R-U 5 U $!,(df  N=f!,(df  N~=f!,(df  N=f) Nzutf-8z.sozmain.cwrbT)binaryr)hashlibsha256encode hexdigestrget_filetempfileTemporaryDirectoryr#r=rNopenwriter include_dirputreadimportlib.utilutilspec_from_file_locationmodule_from_specloader exec_module) srcr%keycache cache_pathtmpdirsrc_pathr]so importlibspecmods rcompile_module_from_srcrs) ..G, - 7 7 9C c "E4& -J  ( ( *fww||FH5Hh$ %KDBb$1"YYqvvxD6dYK  + >> 1 1$ CD .. ) )$ /CKKC J%$  + *s<+-E=E*'E=#E,4E= E) %E=, E: 6E== F c.^\rSrSrU4SjrSrSrU=r$)HIPUtilscn>[US5(d[[U]U5UlUR$)Ninstance)hasattrsuperr__new__r)cls __class__s rrHIPUtils.__new__s-sJ'' 37%>"r)rr)rrrrrrr __classcell__rs@rrrs ? ?rrc@USS:XagSSSSSSS S SS S S S S S S.U$)Nr*hipDeviceptr_tint32_tint8_tint16_tint64_tuint32_tuint8_tuint16_tuint64_tfloatdoublei1i8i16i32i64u1u8u16u32u64fp16bf16fp32f32fp64rtys r ty_to_cpprsQ !u|   !  rc[U5nSRSUR555nSnSnSRUR5Vs/sHo"U"U55PM sn5n SU -n [U5S:a)SSRSUR555-OSn [ 5n UR 5V s/sH oU;dM U PM nn S U S [U5S:aSU-OSS SRS U55S USSRUR5V Vs/sHupU"U5SU S3PM snn 5SU SU SSRUR5V Vs/sHupUSS:XaSU SU SU SU S3 OSPM! snn 5S[U5S:a)SSRSUR555-OSS3nU$s snfs sn fs snn fs snn f)Nz, c3F# UHup[U5SU3v M g7f)z argN)r.0irs r make_launcher..s#SARYr]O4s3ARs!c@USS:XagSSSSSSS S SS S S S S S S.U$)Nrr PyObject*rrrrrrrrrrrrrs r_extracted_type&make_launcher.._extracted_typesQ a5C< ! rc &SSSSSSSSSS S S S . U$) NOr]r\lbhrBHIK) rrrlongrrrrrrrrrrs r format_of make_launcher..format_ofs:     r iiiKKOOOOrc30# UH upSU3v M g7f)z&_argNrrs rrrs L:K5:Ksa; #define __HIP_PLATFORM_AMD__ #include #include #include #include #include // The list of paths to search for the HIP runtime library. The caller Python // code should substitute the search path placeholder. static const char *hipLibSearchPaths[] = {"a"}; // The list of HIP dynamic library symbols and their signature we are interested // in this file. #define HIP_SYMBOL_LIST(FOR_EACH_ERR_FN, FOR_EACH_STR_FN) \ FOR_EACH_STR_FN(hipGetErrorString, hipError_t hipError) \ FOR_EACH_ERR_FN(hipModuleLaunchKernel, hipFunction_t f, \ unsigned int gridDimX, unsigned int gridDimY, \ unsigned int gridDimZ, unsigned int blockDimX, \ unsigned int blockDimY, unsigned int blockDimZ, \ unsigned int sharedMemBytes, hipStream_t stream, \ void **kernelParams, void **extra) \ FOR_EACH_ERR_FN(hipPointerGetAttribute, void *data, \ hipPointer_attribute attribute, hipDeviceptr_t ptr) // The HIP symbol table for holding resolved dynamic library symbols. struct HIPSymbolTable { #define DEFINE_EACH_ERR_FIELD(hipSymbolName, ...) \ hipError_t (*hipSymbolName)(__VA_ARGS__); #define DEFINE_EACH_STR_FIELD(hipSymbolName, ...) \ const char *(*hipSymbolName)(__VA_ARGS__); HIP_SYMBOL_LIST(DEFINE_EACH_ERR_FIELD, DEFINE_EACH_STR_FIELD) }; static struct HIPSymbolTable hipSymbolTable; bool initSymbolTable() { // Use the HIP runtime library loaded into the existing process if it exits. void *lib = dlopen("libamdhip64.so", RTLD_NOLOAD); if (lib) { // printf("[triton] chosen loaded libamdhip64.so in the process\n"); } // Otherwise, go through the list of search paths to dlopen the first HIP // driver library. if (!lib) { int n = sizeof(hipLibSearchPaths) / sizeof(hipLibSearchPaths[0]); for (int i = 0; i < n; ++i) { void *handle = dlopen(hipLibSearchPaths[i], RTLD_LAZY | RTLD_LOCAL); if (handle) { lib = handle; // printf("[triton] chosen %s\n", hipLibSearchPaths[i]); } } } if (!lib) { PyErr_SetString(PyExc_RuntimeError, "cannot open libamdhip64.so"); return false; } // Resolve all symbols we are interested in. dlerror(); // Clear existing errors const char *error = NULL; #define QUERY_EACH_FN(hipSymbolName, ...) \ *(void **)&hipSymbolTable.hipSymbolName = dlsym(lib, #hipSymbolName); \ error = dlerror(); \ if (error) { \ PyErr_SetString(PyExc_RuntimeError, \ "cannot query " #hipSymbolName " from libamdhip64.so"); \ dlclose(lib); \ return false; \ } HIP_SYMBOL_LIST(QUERY_EACH_FN, QUERY_EACH_FN) return true; } static inline void gpuAssert(hipError_t code, const char *file, int line) { if (code != HIP_SUCCESS) { const char* prefix = "Triton Error [HIP]: "; const char* str = hipSymbolTable.hipGetErrorString(code); char err[1024] = {0}; snprintf(err, 1024, "%s Code: %d, Messsage: %s", prefix, code, str ); PyErr_SetString(PyExc_RuntimeError, err); } } #define HIP_CHECK(ans) { gpuAssert((ans), __FILE__, __LINE__); } static void _launch(int gridX, int gridY, int gridZ, int num_warps, int num_ctas, int clusterDimX, int clusterDimY, int clusterDimZ, int shared_memory, hipStream_t stream, hipFunction_t functionz>) { // printf("_launch hip kernel\n"); void *params[] = { c3,# UH nSU3v M g7f)z&argNr)rrs rrrMs 0) { HIP_CHECK(hipSymbolTable.hipModuleLaunchKernel(function, gridX, gridY, gridZ, aw*num_warps, 1, 1, shared_memory, stream, params, 0)); } } typedef struct _DevicePtrInfo { hipDeviceptr_t dev_ptr; bool valid; } DevicePtrInfo; static inline DevicePtrInfo getPointer(PyObject *obj, int idx) { DevicePtrInfo ptr_info; ptr_info.dev_ptr = 0; ptr_info.valid = true; if (PyLong_Check(obj)) { ptr_info.dev_ptr = (hipDeviceptr_t)PyLong_AsUnsignedLongLong(obj); return ptr_info; } if (obj == Py_None) { // valid nullptr return ptr_info; } PyObject *ptr = PyObject_GetAttrString(obj, "data_ptr"); if(ptr){ PyObject *empty_tuple = PyTuple_New(0); PyObject *ret = PyObject_Call(ptr, empty_tuple, NULL); Py_DECREF(empty_tuple); Py_DECREF(ptr); if (!PyLong_Check(ret)) { PyErr_SetString(PyExc_TypeError, "data_ptr method of Pointer object must return 64-bit int"); ptr_info.valid = false; return ptr_info; } ptr_info.dev_ptr = (hipDeviceptr_t)PyLong_AsUnsignedLongLong(ret); if(!ptr_info.dev_ptr) return ptr_info; uint64_t dev_ptr; hipError_t status = hipSymbolTable.hipPointerGetAttribute(&dev_ptr, HIP_POINTER_ATTRIBUTE_DEVICE_POINTER, ptr_info.dev_ptr); if (status == hipErrorInvalidValue) { PyErr_Format(PyExc_ValueError, "Pointer argument (at %d) cannot be accessed from Triton (cpu tensor?)", idx); ptr_info.valid = false; } ptr_info.dev_ptr = (hipDeviceptr_t)dev_ptr; Py_DECREF(ret); return ptr_info; } PyErr_SetString(PyExc_TypeError, "Pointer argument must be either uint64 or have data_ptr method"); return ptr_info; } static PyObject* launch(PyObject* self, PyObject* args) { // printf("launch\n"); int gridX, gridY, gridZ; uint64_t _stream; uint64_t _function; PyObject *launch_enter_hook = NULL; PyObject *launch_exit_hook = NULL; PyObject *kernel_metadata = NULL; PyObject *launch_metadata = NULL;  z _argz; z if(!PyArg_ParseTuple(args, "z", &gridX, &gridY, &gridZ, &_stream, &_function, &kernel_metadata, &launch_metadata, &launch_enter_hook, &launch_exit_hook a=)) { return NULL; } // extract kernel metadata int num_warps, num_ctas, shared_memory, clusterDimX, clusterDimY, clusterDimZ; if (!PyArg_ParseTuple(kernel_metadata, "iiiiii", &num_warps, &num_ctas, &shared_memory, &clusterDimX, &clusterDimY, &clusterDimZ)) { return NULL; } // extract launch metadata if (launch_enter_hook != Py_None){ PyObject* args = Py_BuildValue("(O)", launch_metadata); PyObject* ret = PyObject_CallObject(launch_enter_hook, args); Py_DECREF(args); if (!ret) return NULL; } // raise exception asap rzDevicePtrInfo ptr_infoz = getPointer(_argz); if (!ptr_infoz.valid) return NULL;z; _launch(gridX, gridY, gridZ, num_warps, num_ctas, clusterDimX, clusterDimY, clusterDimZ, shared_memory, (hipStream_t)_stream, (hipFunction_t)_functionc3N# UHupUSS:XaSUS3OSU3v M g7f)rrptr_infoz.dev_ptr_argNrrs rrrsj~l}chcdEGHIEJLOEOmuvwuxx@kAW[\][^U_k_l}s#%an); if(launch_exit_hook != Py_None){ PyObject* args = Py_BuildValue("(O)", launch_metadata); PyObject* ret = PyObject_CallObject(launch_exit_hook, args); Py_DECREF(args); if (!ret) return NULL; } if(PyErr_Occurred()) { return NULL; } // return None Py_INCREF(Py_None); return Py_None; } static PyMethodDef ModuleMethods[] = { {"launch", launch, METH_VARARGS, "Entry point for all kernels with this signature"}, {NULL, NULL, 0, NULL} // sentinel }; static struct PyModuleDef ModuleDef = { PyModuleDef_HEAD_INIT, "__triton_launcher", NULL, //documentation -1, //size ModuleMethods }; PyMODINIT_FUNC PyInit___triton_launcher(void) { if (!initSymbolTable()) { return NULL; } PyObject *m = PyModule_Create(&ModuleDef); if(m == NULL) { return NULL; } PyModule_AddFunctions(m, ModuleMethods); return m; } )r(rNitemsvaluesrckeys) constants signatureids warp_size start_desc arg_declsrrr args_formatformat args_listrrparamsrzs r make_launcherrsYJ SARSSI* ''IDTDTDVWDVb9_R%89DVWXK ; &FPST]P^abPbtyy L)//:K LLLhjI02K#) @)Ai-?a)F @ -.9MS:CfX[\eXfijXjDHKTDTprCssyy UU^T_;`v88Y__=N O=NEA#$E!B /=N OPQR &x(RR[Q\](99R[RaRaRcdRcINIJoqrsotx{o{&qc);A3bCSTUSVVjkBDDRcdeffYBEFOBPSTBTZ^aeajajj~lul{l{l}j~a~Z~Z\Y]*]i^ C~ JOXAz P.dsG- G2G28G7?&G=(c \rSrSrSrSrSrg) HIPLauncheric^S[TS5(aTRRO [50n[TS5(a TRO [ 5nU4SjnUR 5VVs0sHupgU"U5U_M nnnTRR 5VVs0sHupgU"U5U_M nnn[XHX2R5m[TS5n U RUl gs snnfs snnf)Nids_of_const_exprsfnrc|>[U[5(a%TRRR U5$U$N) isinstancestrr arg_namesindex)rrzs r&HIPLauncher.__init__..s/As9K9KCFF,,2215RQRRr__triton_launcher) rr constexprstuplerdictrrrrrlaunch) rrzmetadatarrcst_keyr{valuerrs ` rrHIPLauncher.__init__s#'#t:L:LSVV%6%6RWRYZ%,S+%>%>CMMDF R;D??;LM;LZSWS\5(;L M;>==;N;N;PQ;PZSWS\5(;P QI#7I7IJ%c+>?jj NQs 5C6*C<c(UR"U0UD6 grr)rargskwargss r__call__HIPLauncher.__call__s T$V$rrN)rrrrrrrrrrrrs !%rrcP^\rSrSrU4SjrSr\S5rSrSr Sr Sr U=r $) HIPDrivericV>[TU]5 [5Ul[Ulgr)rrrutilsr launcher_cls)rrs rrHIPDriver.__init__s Z 'rc"SSKnUR$Nr)rAcuda)rrAs rget_device_interfaceHIPDriver.get_device_interfaceszzrc:SSKnURRSL$r )rAversionhip)rAs r is_activeHIPDriver.is_actives}}  ,,rcUR5nURRU5nUSnUSn[SUR S5SU5$)NarchwarpSizerrDr)get_current_devicer rrrP)rdevicedevice_propertiesrrs rget_current_targetHIPDriver.get_current_targetsU((* JJ<>rcSSKJn U$)Nr)do_bench)triton.testingr)rrs rget_benchmarkerHIPDriver.get_benchmarkers +rc\SSKnSnUR[US-5URSS9$)Nrir)dtyper)rAemptyint)rrA cache_sizes rget_empty_cache_for_benchmark'HIPDriver.get_empty_cache_for_benchmarks.' {{3zQ/uyy{PPr)r r ) rrrrrr staticmethodrrr!r)rrrs@rrrs7( --?QQrr) functoolsr#rhrQrmpathlibrtriton.runtime.buildrtriton.runtime.cachertriton.backends.compilerrtriton.backends.driverrr=rrealpath__file__rNrqr> lru_cachercrobjectrrrrrrrrr6s '2., ''//"''**84 5ww||GY/0 -` ;R;R|&?v?( ,Qh %& % Q Qr