
    zbj                        S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKJr  S SK	r	S SK
Jr  SSSSSS	S
SSS.	rSSSS.rS rS rS?S jrS\S\S\4S jrS\S\S\S\4S jrS\S\S\S\S\4
S jr S@S\S \S!\4S" jjrS#\4S$ jr S@S\S \4S% jjr S@S\S\S&\S\S\S'\S(\S\S)\S*\S+\S!\4S, jjr   SAS\S\S\S-\4S. jjr  SBS\S \4S/ jjr  SBS\S\S&\S\S\S'\S(\S\S)\S*\S-\S!\4S0 jjr  SCS1\S2\S\S\S'\S(\S\S)\S*\S3\S4\S5\4S6 jjr   SDS1\S2\S\S\S\S'\S(\S\S)\S*\S3\S4\S5\S!\4S7 jjr    SDS1\S2\S\S\S'\S(\S\S)\S*\S3\S4\S!\4S8 jjr!   SDS1\S2\S\S\S'\S(\S\S)\S*\S3\S4\S!\4S9 jjr" SES\S\S\S\S\S'\S(\S\S)\S*\S!\4S: jjr#S; r$SES< jr%S= r&\'S>:X  a  S SK(r( \&" 5         gg! \) a!    \(RT                  " \RV                  " 5       6    gf = f)F    N)Pathmeasure_memoryzrunwayml/stable-diffusion-v1-5zstabilityai/stable-diffusion-2z stabilityai/stable-diffusion-2-1z+stabilityai/stable-diffusion-xl-refiner-1.0z/stabilityai/stable-diffusion-3-medium-diffusersz'stabilityai/stable-diffusion-3.5-mediumz&stabilityai/stable-diffusion-3.5-largez black-forest-labs/FLUX.1-schnellzblack-forest-labs/FLUX.1-dev)	1.5z2.02.1zxl-1.0z3.0Mz3.5Mz3.5LzFlux.1SzFlux.1DCUDAExecutionProviderMIGraphXExecutionProviderTensorrtExecutionProvider)cudamigraphxtensorrtc                      / SQn SnX4$ )N)
z.a photo of an astronaut riding a horse on marsz@cute grey cat with blue eyes, wearing a bowtie, acrylic paintingzia cute magical flying dog, fantasy art drawn by disney concept artists, highly detailed, digital paintingzdan illustration of a house with large barn with many cute flower pots and beautiful blue sky sceneryzgone apple sitting on a table, still life, reflective, full color photograph, centered, close-up productzWbackground texture of stones, masterpiece, artistic, stunning photo, award winner photozSnew international organic style house, tropical surroundings, architecture, 8k, hdrznbeautiful Renaissance Revival Estate, Hobbit-House, detailed painting, warm colors, 8k, trending on Artstationzcblue owl, big green eyes, portrait, intricate metal design, unreal engine, octane render, realisticzldelicate elvish moonstone necklace on a velvet background, symmetrical intricate motifs, leaves, flowers, 8kz*bad composition, ugly, abnormal, malformed )promptsnegative_prompts     ڶ/tmp/claude-0/-home-danvics-docker-quiz/c1e0577a-e42c-4a3d-b1ea-3edd61103a4e/scratchpad/stt/lib/python3.13/site-packages/onnxruntime/transformers/models/stable_diffusion/benchmark.pyexample_promptsr   &   s    G CO##    c                      g)N)zwarm upbadr   r   r   r   warmup_promptsr   9   s    r   c                     [        SXUS9$ )NT)is_gpufuncmonitor_typestart_memoryr   )r   r   r   s      r   measure_gpu_memoryr   =   s    DZfggr   
model_name	directorydisable_safety_checkerc                 t   SSK JnJn  SS KnUbH  [        R
                  R                  U5      (       d   eUR                  5       nUR                  UUUS9nOUR                  U SUSS9nUR                  UR                  R                  5      Ul
        UR                  SS9  U(       a  S Ul        S Ul        U$ )Nr   )DDIMSchedulerOnnxStableDiffusionPipeline)providersess_optionsonnxT)revisionr$   use_auth_tokendisable)	diffusersr"   r#   onnxruntimeospathexistsSessionOptionsfrom_pretrainedfrom_config	schedulerconfigset_progress_bar_configsafety_checkerfeature_extractor)	r   r   r$   r    r"   r#   r,   session_optionspipes	            r   get_ort_pipeliner:   A   s    Dww~~i((((%446*::( ; 
 +::	 ; 
 #..t~~/D/DEDN   ."!%Kr   enable_torch_compileuse_xformersc                    SU ;   a  SSK Jn  UR                  U [        R                  S9R                  S5      nU(       aL  UR                  R                  [        R                  S9  [        R                  " UR                  SSS	9Ul        U$ S
U ;   a  SSK J	n  UR                  U [        R                  S9R                  S5      nU(       aL  UR                  R                  [        R                  S9  [        R                  " UR                  SSS	9Ul        U$ SSK J
nJn  SSKJn	Jn
  UR                  X
S9R                  S5      nUR                  R                  U	S9  U(       a  UR                  5         U(       az  [        R                  " UR                  5      Ul        [        R                  " UR                  5      Ul        [        R                  " UR                   5      Ul        [#        S5        UR%                  UR&                  R(                  5      Ul        UR+                  SS9  U(       a  S Ul        S Ul        U$ )NFLUXr   )FluxPipeline)torch_dtyper   )memory_formatzmax-autotuneT)mode	fullgraphzstable-diffusion-3)StableDiffusion3Pipeline)r"   StableDiffusionPipeline)channels_lastfloat16z)Torch compiled unet, vae and text_encoderr)   )r+   r?   r1   torchbfloat16totransformerrF   compilerD   r"   rE   rG   unet*enable_xformers_memory_efficient_attentionvaetext_encoderprintr2   r3   r4   r5   r6   r7   )r   r    r;   r<   r?   r9   rD   r"   rE   rF   rG   s              r   get_torch_pipelinerR   _   s   *++JENN+SVVW]^e.A.AB$}}T-=-=N^bcDz)6'77
PUP^P^7_bbcije.A.AB$}}T-=-=N^bcD@,"22:2SVVW]^DIILL}L-779MM$)),	==*!MM$*;*;<9:"..t~~/D/DEDN   ."!%Kr   engine
batch_sizestepsc                     UR                  S5      S   R                  SS5      nU  SU SU SU 3U(       a  S-   $ S	-   $ )
N/zstable-diffusion-sd__b_s _safe)splitreplace)rS   r   rT   rU   r    short_model_names         r   get_image_filename_prefixrb      sV    !'',R0889LdSXQ'(:,b@J`Bnnfmnnr   image_filename_prefixskip_warmupc                   ^ ^^^^^
 SSK Jn  [        T U5      (       d   e[        5       u  pUUU U
UU4S jn[	        XU5      n[	        XU5      nU" 5         / n[        U5       H  u  nnUU:  a    O[        R                  " 5       nT " U/T-  TTTU/T-  S9R                  n[        R                  " 5       nUU-
  nUR                  U5        [        SUS S35        [        U5       H   u  nnUR                  U SU SU S	35        M"     M     SS
KJn  SUTTTTUU[        U5      [        U5      -  [        R                   " U5      UUS.$ )Nr   )r#   c                  R   > T(       a  g [        5       u  pT" U /T-  TTTU/T-  S9  g )Npromptheightwidthnum_inference_stepsr   r   )rh   negativerT   ri   r9   rd   rU   rj   s     r   warmup run_ort_pipeline.<locals>.warmup   s9    )+8j( %%J3	
r   rg   Inference took .3f secondsrZ   .jpg__version__r,   rS   versionri   rj   rU   rT   batch_countnum_promptsaverage_latencymedian_latencyfirst_run_memory_MBsecond_run_memory_MB)r+   r#   
isinstancer   r   	enumeratetimeimagesappendrQ   saver,   ru   sumlen
statisticsmedian)r9   rT   rc   ri   rj   rU   ry   rx   r   memory_monitor_typerd   r#   r   r   rn   first_run_memorysecond_run_memorylatency_listirh   inference_startr   inference_endlatencykimageort_versions   `` ```    `                r   run_ort_pipeliner      so    6d78888.0G

 

 **=|T*+>U
HLw'	6))+8j( %,-
:
 & 	 		/1G$}H56!&)HAuJJ/0!AaS=> * ($ 7   ""|,s</@@$++L9/ 1 r   returnc                     U(       d  U(       a  SU 0O	SU /U-  0O0 n[         R                  R                  5       (       a&  [         R                  " SS9R	                  S5      US'   U$ )Nr   r   )device{   	generator)rH   r   is_available	Generatormanual_seed)r   use_num_images_per_promptis_fluxrT   kwargss        r   get_negative_prompt_kwargsr      sh      ) 0#o%6%CD   zz  #ooV<HHM{Mr   c                 p  ^ ^^^^^
^ [        5       u  pSS Kn[        T UR                  5      mUUUU U
UU4S jn[	        XU5      n[	        XU5      nU" 5         [
        R                  " S5        / n[        U5       H  u  nnUU:  a    O[
        R                  R                  5         [        R                  " 5       n[        USTT5      nT " SU/T-  TTTS.UD6R                  n[
        R                  R                  5         [        R                  " 5       nUU-
  nUR                  U5        [        SUS S35        [        U5       H   u  nnUR                  U SU SU S	35        M"     M     S
[
        R                   TTTTUU[#        U5      [%        U5      -  [&        R(                  " U5      UUS.$ )Nr   c                  l   > T(       a  g [        5       u  p[        USTT5      nT" SU /T-  TT	TS.UD6  g )NFrh   ri   rj   rk   r   r   r   )
rh   rm   extra_kwargsrT   ri   r   r9   rd   rU   rj   s
      r   rn   "run_torch_pipeline.<locals>.warmup  sC    )+1(E7JWqVHz)&[`qdpqr   Fr   rp   rq   rr   rZ   rs   rH   rv   r   )r   r+   r~   r?   r   rH   set_grad_enabledr   r   synchronizer   r   r   r   rQ   r   ru   r   r   r   r   )r9   rT   rc   ri   rj   rU   ry   rx   r   r   rd   r   r   r+   rn   r   r   r   r   rh   r   r   r   r   r   r   r   r   s   `` ```    `                @r   run_torch_pipeliner      s     /0Gy556Gr r **=|T*+>U
H	5!Lw'	6

 ))+1/5'S]^ 
8j( %	

 
 & 	 	

 		/1G$}H56!&)HAuJJ/0!AaS=> *' (. $$ ""|,s</@@$++L9/ 1 r   r$   ri   rj   ry   rx   tuningc                 P   UnU(       a  US;   a  USSS.4n[         R                   " 5       n[        XX5      n[         R                   " 5       n[        SUU-
   S35        [        SXXt5      n[	        UUUUUUUU	U
UUS9nUR                  U UUR                  SS	5      US
S.5        U$ )N)r      )tunable_op_enabletunable_op_tuning_enableModel loading took rr   ortrd   ExecutionProviderr]   Fr   r   r$   r    enable_cuda_graph)r   r:   rQ   rb   r   updater`   )r   r   r$   rT   r    ri   rj   rU   ry   rx   r   r   r   rd   provider_and_options
load_startr9   load_endrc   results                       r   run_ortr   8  s      $(77 (_`*abJJ3G`Dyy{H	: 56h
?@5eZUZsF MM$" (()<bA&<!&	
 Mr   use_io_bindingc                     SSK Jn  Ub5  [        R                  R	                  U5      (       a  UR                  XUS9nO#UR                  U SUUS9nUR                  U5        U(       a  S Ul        S Ul        U$ )Nr   )ORTPipelineForText2Image)r$   r   T)exportr$   r   )	optimum.onnxruntimer   r-   r.   r/   r1   save_pretrainedr6   r7   )r   r   r$   r    r   r   pipelines          r   get_optimum_ort_pipeliner   l  s~     =	!:!:+;;Iiw;x+;;)	 < 
 	  +"&%)"Or   c                   ^ ^^^^^^
^^ [        S[        T 5      5        SSKJn  [	        T U5      m[        5       u  pUUUUU UUU
U4	S jn[        XU5      n[        XU5      nU" 5         [        UT
TT5      n/ n[        U5       H  u  nnUU:  a    O[        R                  " 5       nT
(       a  T " SUTTTTS.UD6R                  nOT " SU/T-  TTTS.UD6R                  n[        R                  " 5       nUU-
  nUR                  U5        [        SUS S	35        [        U5       H   u  nnUR                  U S
U S
U S35        M"     M     SSKJn  SUTTTTTU[        U5      [!        U5      -  ["        R$                  " U5      UUS.$ )NzPipeline typer   )ORTFluxPipelinec            	         >	 T(       a  g [        5       u  p[        UT
TT5      nT
(       a  T" SU TTT	TS.UD6  g T" SU /T-  TTT	S.UD6  g )Nrh   ri   rj   rk   num_images_per_promptr   r   r   )rh   rm   r   rx   rT   ri   r   r9   rd   rU   r   rj   s      r   rn   (run_optimum_ort_pipeline.<locals>.warmup  sq    )+1(<UW^`jk$ $)&1  u:-fE_duhtur   r   r   rp   rq   rr   rZ   rs   rt   optimum_ortrv   r   )rQ   type&optimum.onnxruntime.modeling_diffusionr   r~   r   r   r   r   r   r   r   r   r,   ru   r   r   r   r   )r9   rT   rc   ri   rj   rU   ry   rx   r   r   r   rd   r   r   r   rn   r   r   r   r   r   rh   r   r   r   r   r   r   r   r   s   `` ``` `  ``                 @r   run_optimum_ort_pipeliner     s    
/4:&F/G.0Gv v& **=|T*+>U
H-o?XZacmnLLw'	6))+$ $)&0  f   x*,V5^cgsf  		/1G$}H56!&)HAuJJ/0!AaS=> *+ (0 7   ""|,s</@@$++L9/ 1 r   c                 l   [         R                   " 5       n[        XX$US9n[         R                   " 5       n[        SUU-
   S35        U(       a  U S-   [        U5      R                  -   OU n[        SUX7U5      n[        UUUUUUUU	U
UUS9nUR                  U UUR                  SS5      US	S
.5        U$ )Nr   r   rr   rZ   optimumr   r   r]   Fr   )	r   r   rQ   r   namerb   r   r   r`   )r   r   r$   rT   r    ri   rj   rU   ry   rx   r   r   r   rd   r   r9   r   full_model_namerc   r   s                       r   run_optimum_ortr     s      J#xP^D yy{H	: 56h
?@AJj3&i)=)==PZO5?J7M &F MM$" (()<bA&<!&	
 Mr   work_dirrw   max_batch_sizenvtx_profileuse_cuda_graphc                   ^^^^^- [        S5        SSKJn  U" 5         TU::  d   eSSKJn  U" U5      nUR                  5       nSSKJnJn  SSK	J
n  UR                  nU" U UU5      u  nnnnnU" USUSUUUUUS	9	m-T-R                  R                  UUUS
TTTSSS[        R                  R!                  5       S9  T-R#                  TTT5        UUU-UU4S jn[%        U
UU	5      n[%        U
UU	5      nU" 5         ['        SUTTU5      n/ n[)        5       u  n n![+        U 5       H  u  n"n#U"U:  a    O[,        R,                  " 5       n$T-R/                  U#/T-  U!/T-  TTTSSS9u  n%n&[,        R,                  " 5       n'U'U$-
  n(UR1                  U(5        [        SU(S SU& 35        [+        U%5       H   u  n)n*U*R3                  U SU" SU) S35        M"     M     T-R5                  5         SSKJn+  SSKJn,  0 SUR=                  5       _SS_SU,_SSU+ S3_SU_S T_S!T_S"T_S#T_S$U_S%U_S&[?        U5      [A        U5      -  _S'[B        RD                  " U5      _S(U_S)U_S*U_S+U_$ ),Nzd[I] Initializing ORT TensorRT EP accelerated StableDiffusionXL txt2img pipeline (static input shape)r   init_trt_pluginsPipelineInfo
EngineTypeget_engine_pathsrE   DDIMFr3   
output_dirverboser   r   r   framework_model_direngine_type   T)opt_image_heightopt_image_widthopt_batch_sizestatic_batchstatic_image_shapemax_workspace_size	device_idc                  T   > [        5       u  pTR                  U /T-  U/T-  TTTS9  g N)denoising_stepsr   run)rh   rm   rT   ri   r   rU   rj   s     r   rn   "run_ort_trt_static.<locals>.warmupZ  s3    )+fX
*XJ,CVUdijr   ort_trtg      @r   r   guidanceseedEnd2End took rq    seconds. Inference latency: rZ   rs   rt   r   rS   r,   rw   r$   z	tensorrt()r   ri   rj   rU   rT   rx   ry   rz   r{   r|   r}   r    r   )#rQ   trt_utilitiesr   diffusion_modelsr   
short_nameengine_builderr   r   pipeline_stable_diffusionrE   ORT_TRTbackendbuild_enginesrH   r   current_deviceload_resourcesr   rb   r   r   r   r   r   r   teardownr   ru   r,   r   r   r   r   r   ).r   rw   rT   r    ri   rj   rU   ry   rx   r   r   r   r   r   r   r   pipeline_infor   r   r   rE   r   onnx_dir
engine_dirr   r   rZ   rn   r   r   rc   r   r   r   r   rh   r   r   pipeline_timer   r   r   r   trt_versionr   r   s.     ` ```                                      @r   run_ort_trt_staticr
    s9     

pq /'''- )M))+J;A$$K?OPXZgit?u<Hj*&91 '!%%/
H ""
!**++- #   FE:6k k **=v|T*+>U
H5iZY^`vwL.0G_w'	6))+ (Hz!
*! !- !
 		/1G$gc]*GWX!&)HAuJJ/0!AaS=> *% (* 36m((*- 	; 	i}A.	
 	Z 	& 	 	 	j 	{ 	{ 	3|,s</@@ 	*++L9 	/ 	 1  	!"8!" 	^# r   c                   ^^^^^^1 [        S5        SSKJn  SSKJn  U" 5         TU::  d   eSSKJn  U" U5      nSSKJnJ	n  SSK
Jn  UR                  nU" U UU5      u  nnnnnU" USUS	UUS
US9m1T1R                  R                  UUUSTTTS
S
S	US9  [        T1R                  R!                  5       T1R                  R!                  5       5      nUR#                  U5      u  nnT1R                  R%                  U5        T1R'                  TTT5        UUU1UUU4S jn [)        UU U
5      n![)        UU U
5      n"U " 5         [+        SUTTU5      n#/ n$[-        5       u  n%n&[/        U%5       H  u  n'n(U'U:  a    O[0        R0                  " 5       n)T1R3                  U(/T-  U&/T-  TTTSS9u  n*n+[0        R0                  " 5       n,U,U)-
  n-U$R5                  U-5        [        SU-S SU+ 35        [/        U*5       H   u  n.n/U/R7                  U# SU' SU. S35        M"     M     T1R9                  5         SS Kn0SU0R<                  STTTTU	U[?        U$5      [A        U$5      -  [B        RD                  " U$5      U!U"US.$ )N][I] Initializing TensorRT accelerated StableDiffusionXL txt2img pipeline (static input shape)r   cudartr   r   r   r   r   FT)r3   r   r   r   r   r   r   r   r  r   r  
onnx_opsetr   r   r   r   static_shapeenable_all_tacticstiming_cachec                  d   > T(       a  g [        5       u  pTR                  U /T-  U/T-  TTTS9  g r   r   )rh   rm   rT   ri   r   rd   rU   rj   s     r   rn   #run_tensorrt_static.<locals>.warmup  s9    )+fX
*XJ,CVUdijr   trtr   )r   r   r   rq   r   rZ   rs   r   default)rS   rw   r$   ri   rj   rU   rT   rx   ry   rz   r{   r|   r}   r   )#rQ   r   r  r   r   r   r   r   r   r   r   rE   TRTr   load_enginesmaxmax_device_memory
cudaMallocactivate_enginesr  r   rb   r   r   r   r   r   r   r  r   ru   r   r   r   r   )2r   rw   r   rT   r    ri   rj   rU   ry   rx   r   r   r   r   r   rd   r  r   r   r  r   r   rE   r   r  r  r   r   r  r  rZ   shared_device_memoryrn   r   r   rc   r   r   r   r   rh   r   r   r  r   r   r   r   r  r   s2      ` ```       `                                 @r   run_tensorrt_staticr    s   $ 

ij /'''- )M;A..KJZ-KGHj*&9<
 '!%	H !!/! ! "  H,,>>@(BRBRBdBdBfg$//0ABA%%&:; FE:6k k **=v|T*+>U
H5eZUZ\rsL.0G_w'	6))+ (Hz!
*! !- !
 		/1G$gc]*GWX!&)HAuJJ/0!AaS=> *# ((  ?? ""|,s</@@$++L9/ 1+ r   c                   ^ ^^^^^^^^^*^+^,^-^.^/ [        S5        SS KnSSKJn  SSKJn  Tm,Tm-T,S-  S:w  d	  T-S-  S:w  a  [        ST, ST- S35      eU" 5         TT::  d   eSS	KJn  SS
K	J
m*Jm+  U*UU+UUUUUU 4	S jnSSKJn  U" U5      nU" UU5      m.[        T.R                  R!                  5       T.R                  R!                  5       5      nUR#                  U5      u  nnT.R                  R%                  U5        T.R'                  T,T-T5        SU,U-U.U4S jjm/UU/U4S jn[)        U
UU	5      n[)        U
UU	5      nU" 5         UR+                  5       n[-        SUTTU5      n/ n[/        5       u  nn [1        U5       H  u  n!n"U!U:  a    O[2        R2                  " 5       n#T/" U"/T-  U /T-  SS9u  n$n%[2        R2                  " 5       n&U&U#-
  n'UR5                  U'5        [        SU'S SU% 35        [1        U$5       H   u  n(n)U)R7                  U SU! SU( S35        M"     M     T.R9                  5         USUR:                  STTTTUU[=        U5      [?        U5      -  [@        RB                  " U5      UUTS.$ )Nr  r   r  r      zCImage height and width have to be divisible by 8 but specified as: z and .r   r   c                    >	 T	R                   nT" TX5      u  p4pVnU " USUSTTTUUS9	nUR                  R                  UUUST
TTSSSUS9  U$ )Nr   Fr   r   Tr  )r  r   r  )pipeline_classr  r   r  r  r   r   r  r   r   rT   r   ri   r   r   r   rj   r   s            r   init_pipeline-run_tensorrt_static_xl.<locals>.init_pipelineH  s     nnN^mO
Kj|
 "!%)) 3#

 	%%! 3%#!$% 	& 	
 r   r   c           
      .   > TR                  U UTTTSUS9$ Ng      @r   r   )rh   r   r   image_heightimage_widthr   rU   s      r   run_sd_xl_inference3run_tensorrt_static_xl.<locals>.run_sd_xl_inferencex  s.    ||!  
 	
r   c                  P   > T(       a  g [        5       u  pT" U /T-  U/T-  5        g Nrl   rh   rm   rT   r,  rd   s     r   rn   &run_tensorrt_static_xl.<locals>.warmup  ,    )+VHz1H:
3JKr   r  r   r   r   rq   r   rZ   .pngr   r  r   rS   rw   r$   ri   rj   rU   rT   rx   ry   rz   r{   r|   r}   r   r/  )"rQ   r   r   r  r   r   
ValueErrorr   r   r   r   r   r   rE   r  r   r  r  r  r  r   r   rb   r   r   r   r   r   r  ru   r   r   r   r   )0r   rw   rT   r    ri   rj   rU   ry   rx   r   r   r   r   r   rd   r  r  r   r   r%  rE   r  r  rZ   r  rn   r   r   r   rc   r   r   r   r   rh   r   r   r  r   r   r   r   r   r   r*  r+  r   r,  s0   ` ` ```    ````                           @@@@@@r   run_tensorrt_static_xlr7  !  s   " 

ij. LKa1a1 4QR^Q__depdqqrs
 	

 '''-;! !F B )M4mDHH,,>>@(BRBRBdBdBfg$//0ABA%%&:; L+zB	
 	
L **=v|T*+>U
H##%J5eZUZ\rsL.0G_w'	6))+ 3VHz4IOK\_iKips t		/1G$gc]*GWX!&)HAuJJ/0!AaS=> * (  !?? ""|,s</@@$++L9/ 1+ r   c                   ^^^^^^%^& SSK Jn  SSKJn  U" UUR                  U TTUUTS9m%TU::  d   eT%R                  TTT5        SUU%UU4S jjm&UU&U4S jn[        U
UU	5      n[        U
UU	5      nU" 5         T%R                  R                  5       n[        SUTTU5      n/ n[        5       u  nn[        U5       H  u  nnUU:  a    O[        R                  " 5       nT&" U/T-  U/T-  SS	9u  nn[        R                  " 5       nUU-
  nUR                  U5        [        S
US SU 35        [        U5       H.  u  n n!U SU SU  S3n"U!R                  U"5        [        SU"5        M0     M     T%R!                  5         SSKJn#  SSKJn$  USU$SU# S3TTTTUU[)        U5      [+        U5      -  [,        R.                  " U5      UUUS.$ )Nr   )initialize_pipeline)r   )rw   r   r   ri   rj   r   r   r   c           
      .   > TR                  U UTTTSUS9$ r(  r)  )rh   r   r   ri   r   rU   rj   s      r   r,  +run_ort_trt_xl.<locals>.run_sd_xl_inference  s.    ||!  
 	
r   c                  P   > T(       a  g [        5       u  pT" U /T-  U/T-  5        g r/  rl   r0  s     r   rn   run_ort_trt_xl.<locals>.warmup  r2  r   r   r   r3  r   rq   r   rZ   r4  zImage saved tort   r,   r   r   r5  r/  )
demo_utilsr9  r   r   r   r  r   r  r   rb   r   r   r   r   rQ   r   r  r   ru   r,   r   r   r   r   )'r   rw   rT   r    ri   rj   rU   ry   rx   r   r   r   r   r   rd   r9  r   rn   r   r   r   rc   r   r   r   r   rh   r   r   r  r   r   r   r   filenamer	  r   r   r,  s'     ` ```       `                      @@r   run_ort_trt_xlr@    s   " /)"&&%%!	H '''FE:6	
 	
L **=v|T*+>U
H'',,.J5iZY^`vwL.0G_w'	6))+ 3VHz4IOK\_iKips t		/1G$gc]*GWX!&)HAu/0!AaS=HJJx "H- * ( 36 !{m1- ""|,s</@@$++L9/ 1+ r   c                 b   S[         R                  R                  l        S[         R                  R                  l        [         R
                  " S5        [        R                  " 5       n[        XX45      n[        R                  " 5       n[        SX-
   S35        [        SXXr5      nU(       d2  [         R                  " 5          [        UUUUUUUU	U
UUS9nS S S 5        O[        UUUUUUUU	U
UUS9nWR                  U S U(       a  SO
U(       a  SOS	USS
.5        U$ ! , (       d  f       N9= f)NTFr   rr   rH   r   rL   xformersr  r   )rH   backendscudnnenabled	benchmarkr   r   rR   rQ   rb   inference_moder   r   )r   rT   r    r;   r<   ri   rj   rU   ry   rx   r   r   rd   r   r9   r   rc   r   s                     r   	run_torchrH    s*    $(ENN %)ENN"	5!JjBVeDyy{H	 56h
?@5gzW\u!!#'%#'F $# $!#
 MM$%9	\z_h&<!&	
 MM $#s   D  
D.c                     [         R                  " 5       n U R                  SSS[        S/ SQSS9  U R                  SS	S[        S
[	        [
        R                  5       5      SS9  U R                  SSSSS9  U R                  SSS[        [	        [        R                  5       5      SSS9  U R                  SSS[        S SS9  U R                  SSS[        SSS9  U R                  SSSSS 9  U R                  SS!9  U R                  S"SSS#S 9  U R                  SS$9  U R                  S%SSS&S 9  U R                  SS'9  U R                  S(SSS)S 9  U R                  SS*9  U R                  S+SSS,S 9  U R                  SS-9  U R                  S.S/[        S0/ S1QS2S39  U R                  S4S[        S5S6S9  U R                  S7S[        S5S8S9  U R                  S9S:S[        S;S<S9  U R                  S=S>S[        S?S@S9  U R                  SASBS[        [        S0SC5      SDSES9  U R                  SFSGS[        [        S0SH5      SISJS9  U R                  SKSLSSSMS 9  U R                  SSN9  U R                  5       nU$ )ONz-ez--engineFr,   )r,   r   rH   r   z-Engines to benchmark. Default is onnxruntime.)requiredr   r  choiceshelpz-rz
--providerr   z8Provider to benchmark. Default is CUDAExecutionProvider.z-tz--tuning
store_truezCEnable TunableOp and tuning. This will incur longer warmup latency.)actionrL  z-vz	--versionr   z>Stable diffusion version like 1.5, 2.0 or 2.1. Default is 1.5.)rJ  r   rK  r  rL  z-pz
--pipelinez[Directory of saved onnx pipeline. It could be the output directory of optimize_pipeline.py.)rJ  r   r  rL  z-wz
--work_dirr"  z?Root directory to save exported onnx models, built engines etc.z--enable_safety_checkerzEnable safety checker)rJ  rN  rL  )enable_safety_checkerz--enable_torch_compilez#Enable compile unet for PyTorch 2.0)r;   z--use_xformerszUse xformers for PyTorch)r<   z--use_io_bindingzUse I/O Binding for Optimum.r   z--skip_warmupz
No warmup.r   z-bz--batch_sizer   )r            r!  
          z)Number of images per batch. Default is 1.)r   r  rK  rL  z--heighti   z$Output image height. Default is 512.z--widthz#Output image width. Default is 512.z-sz--steps2   zNumber of steps. Default is 50.z-nz--num_promptsrS  z!Number of prompts. Default is 10.z-cz--batch_count      z(Number of batches to test. Default is 5.z-mz--max_trt_batch_sizerT  rR  zdMaximum batch size for TensorRT. Change the value may trigger TensorRT engine rebuild. Default is 4.z-gz--enable_cuda_graphz/Enable Cuda Graph. Requires onnxruntime >= 1.16)r   )argparseArgumentParseradd_argumentstrlist	PROVIDERSkeys	SD_MODELSset_defaultsintrange
parse_args)parserargss     r   parse_argumentsrg  d  st   $$&F
?<   Y^^%&G   R	   Y^^%&M   j   N   !$	   e4
 2	   U3
'	   U+
+	   u-
	   E*
+8   3   2   .   0   a7   as   >   %0DKr   c                    ^ SS K nUR                  [        R                  " 5       5      nUR	                  5        H;  mU (       a  [        U4S jS 5       5      (       d  M&  [        TR                  5        M=     g )Nr   c              3   @   >#    U  H  oTR                   ;   v   M     g 7fr/  )r.   ).0xlibs     r   	<genexpr>)print_loaded_libraries.<locals>.<genexpr>  s     )`A_Asxx-A_s   )libculibnvr   )psutilProcessr-   getpidmemory_mapsanyrQ   r.   )cuda_related_onlyrq  prl  s      @r   print_loaded_librariesrx    sI    ryy{#A}}!c)`A_)`&`&`#((O r   c                     [        5       n [        U 5        U R                  S:X  a  U R                  S;   a  S[        R
                  S'   SSKJn  SSKJn  U R                  (       ag  U R                  S:X  a  U R                  S;   a  U R                  b  [        S	5      eUR                  U5      UR                  S
5      :  a  [        S5      e[        R                  " S[        R                   SS9  Sn[#        US 5      n[        SU5        [$        U R                     n[&        U R                     nU R                  S:X  GaT  U R                  S:X  GaC  SU R                  ;   a  [        S5        [)        U R*                  U R                  U R,                  SU R.                  U R0                  U R2                  U R4                  U R6                  UUU R8                  SU R                  U R:                  S9nGOC[        S5        [=        U R*                  U R                  U R,                  U R>                  (       + U R.                  U R0                  U R2                  U R4                  U R6                  UUU R8                  SU R                  U R:                  S9nGOU R                  S:X  a  US:X  a  [A        UU R                  UU R,                  U R>                  (       + U R.                  U R0                  U R2                  U R4                  U R6                  UUU RB                  U R:                  S9nGOU R                  S:X  a  U R                  (       a.  [        RD                  RG                  U R                  5      (       d   S5       e[        SU SU RH                   35        [K        UU R                  UU R,                  U R>                  (       + U R.                  U R0                  U R2                  U R4                  U R6                  UUU RH                  U R:                  S9nGOU R                  S:X  a  SU R                  ;   a  [        S5        [M        U R*                  U R                  U R,                  SU R.                  U R0                  U R2                  U R4                  U R6                  UUU R8                  SU R                  U R:                  S9nGOgU R                  S:X  a  [        S5        [O        S;0 S U R*                  _S!U R                  _S"U_S#U R,                  _S$S_S%U R.                  _S&U R0                  _S'U R2                  _S(U R4                  _S)U R6                  _S*U_S+U_S,U R8                  _S-S_S.U R                  _S/U R:                  _6nO[        S0U RP                   S1U RR                   S235        [U        UU R,                  U R>                  (       + U RP                  U RR                  U R.                  U R0                  U R2                  U R4                  U R6                  UUU R:                  S39n[        U5        [W        S4S5S6S79 n/ S8Qn	[X        RZ                  " XS99n
U
R]                  5         U
R_                  U5        S S S 5        U R2                  S::X  a  [a        U R                  S;   5        g g ! , (       d  f       N8= f)<Nr,   )r   1ORT_DISABLE_TRT_FLASH_ATTENTIONr   )rw   rt   )r   r   z:The stable diffusion pipeline does not support CUDA graph.z1.16z.CUDA graph requires ONNX Runtime 1.16 or laterz%(funcName)20s: %(message)sT)formatlevelforcer   z&GPU memory used before loading models:r   xlzNTesting Txt2ImgXLPipeline with static input shape. Backend is ORT TensorRT EP.F)r   rw   rT   r    ri   rj   rU   ry   rx   r   r   r   r   r   rd   zLTesting Txt2ImgPipeline with static input shape. Backend is ORT TensorRT EP.r   r   )r   r   r$   rT   r    ri   rj   rU   ry   rx   r   r   r   rd   z?--pipeline should be specified for the directory of ONNX modelsz/Testing diffusers StableDiffusionPipeline with z provider and tuning=)r   r   r$   rT   r    ri   rj   rU   ry   rx   r   r   r   rd   zGTesting Txt2ImgXLPipeline with static input shape. Backend is TensorRT.zETesting Txt2ImgPipeline with static input shape. Backend is TensorRT.r   rw   r   rT   r    ri   rj   rU   ry   rx   r   r   r   r   r   rd   zNTesting Txt2ImgPipeline with dynamic input shape. Backend is PyTorch: compile=z, xformers=r"  )r   rT   r    r;   r<   ri   rj   rU   ry   rx   r   r   rd   zbenchmark_result.csvar]   )rB   newline)r   r   rS   rw   r$   r    ri   rj   rU   rT   rx   ry   rz   r{   r|   r}   r   )
fieldnamesr   r   )1rg  rQ   rS   rw   r-   environ	packagingr,   ru   r   r$   r   r6  parseloggingbasicConfigINFOr   r`  r^  r@  r   rT   ri   rj   rU   ry   rx   max_trt_batch_sizerd   r
  rO  r   r   r.   isdirr   r   r7  r  r;   r<   rH  opencsv
DictWriterwriteheaderwriterowrx  )rf  rw   r   r   r   sd_modelr$   r   csv_filecolumn_names
csv_writers              r   mainr    s   D	$K{{m#<<7" =@BJJ89%:!!KK=0T]]FZ5Z_c_l_l_t !]^^}}[)GMM&,AA !QRR<GLLX\] %&94@L	
2LA&H'H{{m#(C4<<bc#??'+{{jjjj ,, ,,)$7#66"#55 ,,F$ `a'??+/+E+E'E{{jjjj ,, ,,)$7#66"#55 ,,F" 
		!h2I&I mm'+'A'A#A;;****((((% 3..((
  
	%}}t}}!=!= 	
M	
= 	?zI^_c_j_j^klmmm'+'A'A#A;;****((((% 3;;((
  

	"tt||';WX']]LL#';;****((((% 32211((
" 

	"UV$ 
]]
LL
  
 	

 $(
 ;;
 **
 **
 ((
 ((
 &
 !4
  22
 
  11
  ((!
& 	\]a]v]v\w  xC  DH  DU  DU  CV  VW  X	
 '+'A'A#A!%!:!:**;;****((((% 3((
  
&M	$3	;x
& ^^HF
 F#- 
<2 zzQt}}0DDE 3 
<	;s   :Z::
[__main__r/  )F)r   TF)FF)FT)FTF)T),rY  r  r  r-   r   sysr   pathlibr   rH   benchmark_helperr   r`  r^  r   r   r   r\  boolr:   rR   rb  rb   r   dictr   r   r   r   r   r   r
  r  r7  r@  rH  rg  rx  r  __name__	traceback	Exceptionprint_exceptionexc_infor   r   r   <module>r     s    
  	  
    + ,+-;=541-
	 $++	$&h  X\ <*3 * *\` *pt *Zoc os o oTW oqu o  HH H HVcg < FF Fn 111 1 	1
 !1 1 1 1 1 1 1 1n %#'  !	
 L $WW WN !222 2 	2
 !2 2 2 2 2 2 2 2D @@@ @ !	@
 @ @ @ @ @ @ @ @b !EEE E 	E
 !E E E E E E E E E  !Ej SSS S !	S
 S S S S S S S SF eee e !	e
 e e e e e e e ej BBB !B 	B
 B B B B B B BJl^BFJ z3	 
  3!!3<<>23s   1F: :$G! G!