From 6a333a4357677807260accef9ce35227eb798872 Mon Sep 17 00:00:00 2001
From: "William S. Moses" <gh@wsmoses.com>
Date: Mon, 4 Jul 2022 01:51:59 -0400
Subject: [PATCH] Add mcustom

---
 {6_multisource => 6_multisource_lto}/Makefile |   0
 .../multisource.c                             |   0
 {6_multisource => 6_multisource_lto}/myblas.c |   0
 {6_multisource => 6_multisource_lto}/myblas.h |   0
 7_multisource_custom/Makefile                 |  13 +++++
 7_multisource_custom/lib.exe                  | Bin 0 -> 18800 bytes
 7_multisource_custom/multisource.c            |  50 ++++++++++++++++++
 7_multisource_custom/myblas.c                 |  33 ++++++++++++
 7_multisource_custom/myblas.h                 |  29 ++++++++++
 docker/Dockerfile                             |   4 +-
 10 files changed, 127 insertions(+), 2 deletions(-)
 rename {6_multisource => 6_multisource_lto}/Makefile (100%)
 rename {6_multisource => 6_multisource_lto}/multisource.c (100%)
 rename {6_multisource => 6_multisource_lto}/myblas.c (100%)
 rename {6_multisource => 6_multisource_lto}/myblas.h (100%)
 create mode 100644 7_multisource_custom/Makefile
 create mode 100755 7_multisource_custom/lib.exe
 create mode 100644 7_multisource_custom/multisource.c
 create mode 100644 7_multisource_custom/myblas.c
 create mode 100644 7_multisource_custom/myblas.h

diff --git a/6_multisource/Makefile b/6_multisource_lto/Makefile
similarity index 100%
rename from 6_multisource/Makefile
rename to 6_multisource_lto/Makefile
diff --git a/6_multisource/multisource.c b/6_multisource_lto/multisource.c
similarity index 100%
rename from 6_multisource/multisource.c
rename to 6_multisource_lto/multisource.c
diff --git a/6_multisource/myblas.c b/6_multisource_lto/myblas.c
similarity index 100%
rename from 6_multisource/myblas.c
rename to 6_multisource_lto/myblas.c
diff --git a/6_multisource/myblas.h b/6_multisource_lto/myblas.h
similarity index 100%
rename from 6_multisource/myblas.h
rename to 6_multisource_lto/myblas.h
diff --git a/7_multisource_custom/Makefile b/7_multisource_custom/Makefile
new file mode 100644
index 0000000..bd29b39
--- /dev/null
+++ b/7_multisource_custom/Makefile
@@ -0,0 +1,13 @@
+all: lib.exe
+
+clean:
+	rm -f *.o *.ll *.exe
+
+%.o: %.c
+	../dockerscript.sh clang-12 -c /host/$^ -O2 -Xclang -load -Xclang /Enzyme/enzyme/build/Enzyme/ClangEnzyme-12.so -ffast-math -o /host/$@
+
+lib.exe: myblas.o multisource.o
+	../dockerscript.sh clang-12 /host/myblas.o /host/multisource.o -O2 -Xclang -load -Xclang /Enzyme/enzyme/build/Enzyme/ClangEnzyme-12.so -ffast-math -o /host/$@
+
+run-%: %.exe
+	../dockerscript.sh /host/$^ 3
diff --git a/7_multisource_custom/lib.exe b/7_multisource_custom/lib.exe
new file mode 100755
index 0000000000000000000000000000000000000000..0ab4b2bd58b92219b39720e3652104615367ecf8
GIT binary patch
literal 18800
zcmeHPeQ;dWb-()|TgLY8%3oSO8b4rNWI(H3SqK@&_^mA2Pp~$MErSR)>(zcpTUNX3
zet=Dht5m@B1w@>dHk}MJ^&g#<{KNPUQV7$4WEtChG}ts@Af%}QLsx;=LSV-M+uPr{
z?_O#5NeY?iOlO+eYrlK$`Q3BQJ@>qK-`@M~d-}`v;3liZ!ep|tTN!Z&Tq=i*oUs8}
z0IISUwh+08UCZWyw{r{`r)nAPvvQV6T5Yh*21<HQi~6UnDmda*BJ3)jPgCh3QPP`E
zswf(?uF7B~WiEp!JzgbvCi*97yKqd>Qa#?DV%TAdY7_*mqmCd+TB=8}srK?}R(-3=
z3zBN0-<^U=Jz_~u5qgTylk~U<grqW_<Rcx2g}uX86&&#@5!S}@X-cw`IzP-ssXU+%
zM^r7NPjZeiNu}O5phtfGp-F~)n`m#YxSSXLlT@&*A(`lDZfZzI>ywF8eoy_Lmgf5A
zCT}+FZRGu421n>rY~H$q{n_81_@{Tad}pv~!>yMcbuRRM{cpPdoctyq$R5d1B6%5p
z!sS7Ae`vZKf=gx*UN{GS-5mH$bKu*7D>%$a0We#;blJ>?GsG`K;c8}~P&k{7Wpbf-
zIFV!nnM5iVXW3jPmrk<&SbsK_WBuV|G97^=X@#=6Fr@p#i4@x$>}cH>YV<aGZ$fQv
ze>x><Lul73{uw0zNEZIjh&}Q7D-sK+elsyh6B|@c<$Z)R$j+DGVo38Ux2keFA32>b
znUoS-o>Qu;E5XH3<tn}sy!1M5DZ#0YN~%Pl5`jttDiNqepb~*f1S%1zMBtAp0-rg*
z_Jua|NwxM^)p!GA+9M-5Yw@@?bhvtqce8lY`+$q9P9b-$R8b<nkLt!x7Liu{f_R#W
zk00m!o5a%;e0+rSuM$sF@9~42KSVq&$HotUr}mxC?DSmtcF=P{9YiC~ba^g}#hoi3
zh6rO+@BdAQe&SwDe_tCqHPPACadgDzR+)BmbflKZQFR&oT7C`ee!idN$Kqb8zW@17
zOz5@sl;@T{PZp_*3tl3icrw0!EIutv{d<wifBQ5i`h+%mdV@B4-lkcOXeTb^YNjmA
z7Z%L=)c)Z5v$PD$FWsT(9~K@&FOO7x0K~FuY({$||M7RGeR$-}eYFAo_|O;j?25Sa
zt`Yw$$X^+6K!8C3jJ4^bL+@RPw>$4T_ku$G(T_poO%lWL?zAcA$~MO0ZBEzD+LLjo
zD_T22%(7aPbG0rQsCx~-ve&6}h)PG0E&nBDZ$ajv6(>=2C09&<X<9+$=n7ZdS$8xZ
za4vfpg^#Fk3fc0Dlzk4UfWqD5knKJT?i1n)acB3>!JHxHV^r;=s?)^%I_}*0b1M9f
zvI)v?=DX3K?l*{eABA7y_?0$vNY#c$E;t{#5zz$l4otijcP@Wz(%JnQM7!ZJI^kS?
z2*9D>+R@y-9jlLNqwm>)pO}uxe)MIL6WLV`RD2M3-u)gF4ny(Oq;n@IVaq?51P=NN
zD7Ek^j@PxJQR@APj-ioa{`oQI%0UF5ZP)ZKHT}1m{+6b{8q`PI^<xDZ9hz>Nbk=OF
zMs~YP)8D>_E@_;P&fsHR9=G;bo5wZjbTJ%Sbx>dCDP*D4J+>b$j^IGc-J@uOeSB%!
zdAdH$JjQNayFS{XAJg;`nm(!Nm$c!BJg!0nfs)O%dE96y=ut4P>q<rk>_gv8E-7%d
z%|q@6JuYn+#lW`C!d5cEb>x8REPV4~vDiJ{g&Oi_N8ql&oq-*JuI-(HmuQp^6|D~~
z)gIed?LYUzW#9{6d|51Z=ts9i&UB1^Xb<XV3g>}f6gnSPaGPj<q^Rk3&pub1elajq
zwB$AayW{k<i6nn2C7qD$&<_*#g2a{q)ATXI{#jy|bGettlNc}h_|SWoH2n}R9=HFT
z|6Lyc;e8eiqDklVN423bE5;0pIK?%s1!T8P3>~o<)klaukIV{nMbm$!t$indNImzm
zC3o%c?ZU?B@a@9HE86G>c1YV1=$fD#r}RvrMmn_(&cOK&{dC9ZM|Mp=>OU9MCkrRO
zC>BX>9F&J+BjnH_9tFF9<l~*Z2QOSesNw&J;s1luf5Z2mQlp<yqn}k9o*x?CC!Bf%
zaNB3N&QR&b@IhCO*oO{X($<dVpW|J*?#DQp#nq#R=IHPuy2&6S=*~&r9lD5vHr}09
z>E@4cays;vr~0vgDjvJhqv>aY`bV1ni=aN<uAeV_%H@s}eh6x++oFG3Sc2Y82AnRt
z=cVn6$_`oz2Y!c+j?iz<J%OEpU4eT8p<QDysi=I>h3v;3WG~PIx5A^4d}&>I#n-5!
zpDKdo6|1PCiz+skS6ojO>rgR0#P-v8pgz;^qybU*3xpqoXZgE$(hADQ2!;~IG_Ih-
z?(zFjU#@Q+k8S!Qk1;0SpesTQebbnf<BN|+{g~0X;Ye_}%TtXp-Wk|d_z78{0V(C_
z_aksuaCrT?KV>X%XNUe)U`L03HqaH&FYM5U>ze>?53arlyVV8fd1#MbwB=UPXc$EI
zG0G-``sp_PzXQc(|Air8(boQF{@45*?2;#;^f!+GzCw>E*(6D0ra;yzsS<%o1S%1z
zM4%FZN(BBtM8JZ*xal;pp`kCG%{BDrlet7TozFyK-U!16p+dWVbtv4ZMEb%R<@)Pa
zv))WN8VV>^uUC9Vxz#9d2A+sPCz{TMd$M(b)%W;zDeIM0ajXX{SK8Lm?7jxa!866;
z+n}zAVv#oUe4q!x?*=^#s(e~3x`7{`EEaDBWuIaC0;+)SCV9{ts0#Ympd+9MK#zky
z4|?ESv1mv4PJ&(qs{FQCYy#~B?FJnIJpf9H_Kzu99^A$(dt8<)7tOEU4;!>+Oz$G>
zfv<SLfDQEdF2Z~`o&+ZMCOuSz<M?U#2Fe_+O%C^M&V>(D53&tcth;`V=Nf{kog|KG
z=++V-64`v1%INzK(Jm!7j(@=M6yiX0RTgl#hOB)K+Vdwn&*C_Ux|N_*w+l67=XD&<
z17A!y5@9ECJPS;8m+OBHjPxZ2mg9@iz8f)LL;&n=f?W+o55)uc%cKCj)!}Y+D6NjV
zR);U(XbCvfq4|%`8?Ji7{$1OnR@?1X!hF0QQjm289MzB_-v?mp$<K>L+V^H{4kd{;
z2&eP-Z@_Oha9;+vf-;@gQ^4yCT<PYmcA(Zs6#K)FnSe};piK_fcdZ=`_Y<~uhcaw$
zcho&z)#mU$I!|-749(x{P*dEy0ND}VwZa<3Y7zS9K0AGG$y_B>B2bAyB?6TQR3cD`
zKqUf|2vj2Q#}t9V8kKivP`nr2DR@k&d6JZ`2?!B#a*fe~9a&2Bc%G7c&-qG&<@0JP
zULkn5c(0k(R+QvC{_Le<n)5c5i1E%gXhHDuUClbd(-sq@9)sY$bR*H)iPCa|Oy#si
z#gnRXLQ5G+dkn(YAXJp^e#$l4LA%N&Bt9s3>9>#9GKQC1d6Mm4DjNOSE#$G?!uu!V
ziRB*WZxi)G{r`-}_sY!;HjDGSNzgk5jSG6epk}?z8#msf)a~fWr*e5^t+&bRtM}(Q
z=ik@p^ZJ^+{?!J~gis_IPW3AH$1>SOI;Hp<y*{tM{)QVGeT{y<uW_y4=U>y<ctboE
zZSt>aY7DnD#_Rq2VtWS2o*~lM=x^~idz;ve@Wwcy#%Ylkvpk7rXLkdzbFkVn47zA3
zvO73Xo>>FnE!!<-gO;auqQXXlz;dsP%JXpWWVaiailxS~%CU5z&2gn;M{SM81z)UQ
zi_LM%!gUMRF1+sY8q1RTs{8w`ms`-dt%=E+j-}Sx=UO3c_ppTvZ-oJv&El}}=L{Ch
zmyX5Ef&>VM#p`V_{%120dRe|V5^0E}Q}IM^Lt;%!Gh>lVE*{S2`op<CJkueyDhpd+
zm0hy^2eVY!Y?s|u<(`Mayt|jq`>QI;H?4Sq+mcUZ6TPWeRKWuv*1so`&9JX%uA#g&
zyi<<1&9n3pd9IH&^yCxCXahW~Ut1rze@(t!GW>!O)p)}{)Z8Rq_wP;RMW^WPePyHn
zX5%7K>id)FRBwGWo$pD;>RWsOa`{wjeQzuk%OoOT!<k6mdgF5erM_4GzoSv9mq(oY
zlYki`Zc2STm9EEI?~(iJ2QukgERsuS01l8nbYpKS(w9l6()nzCKJ`E%6|Ki}u4F8W
zN<#^kbhb~b(p6m_kHkxOoGyyLF2<0gR`$J|EPqebEdchU{W5j%>VK5b_eB+n!%A6_
zXH&|QRelv?!&ULLRh=Pb!=O4mD{f<jS#dk7w#{az3g5}hhR>V2zq68L!(Di``=SlM
z7V>^tB94)<fXVxEIer<F_wRBXzZSFH$82m7Q)a~%Ga2A=I}Rp)|H^SEoBCd;q{4<@
zL;0Oe2@L;hSoPIvDa6Y*wq)w}w6xO3mNNM}T8=Mcv)$Ki_%)Q@Ta?SMz(Z8E6yjwY
zs|7q8?q>9UlsVa^e*Md9Y*WAbW?aq=%($EtnDIX;2LxwhwD&(V*%*H^I-|nIlo=R}
zM?3qpXs*B+#MM82-sMjTc`}=Posd6ix_;>SE!lC2c@<cm)*~le&bI{2wxeLd6JQx<
z@{dvv4$Ca(T?fwnG}-?<E<f}7c^3F={{KYS8Cj~L#Mm01Gxjr3vd{3Y9i{7d&e$iQ
zzt2u*t;!{D7V=+!@;C_mCV|u2AcXH1IK`3DTyduT01L*8=USD^m1eW~a|;jPk{RRW
zc5Y|pc<B=Iavm$~_i}vZ_<fMunK_PV4`B)VMQ<!plKTY*Brf79=hcl?#*|sc-AU+@
zo&Dnc$~}UA5q4B@zW6=?WS;}>f*pDN0ODz$kl!!l<^25zgj;9c$6n;}QhS=dIS2lm
zIdBX5MgAWY?Skd0hU3$AWyuGe{5Ri!)(blaOm?;aSEO<Y-8qN-0pK;(TBfOLi8N#1
zBAiX%w~gr`V~=w?iy6IgN{R0=fIh`>4wYq_5C<2=Z@0<*L6WyFX2-?-L0%8d!u|n)
zHwbITfRi2bdHk7>9}sbnce+yor*~p0@x2<*ap2Tnvz@;udFw2{gTHSAhq`Pok2hx{
zEE>zidK1}PEECH0ha#Aa#j^M^Fq#haCeuCPWGITcQ#KUN?_rU2|3EU9i$%RHzGnaI
zLZNsfl?a71neg5awh%LWSv(W&kA<T7{{FpCDd9qp%$14u@9n|F3r@uod&2qNei#TD
zSSW(MNP=n4lJ+Lc@bcZx*`%<IIhVHgX1KL!TVP9jsC{c&C<Ly|u_z0*-MuxirDJ1x
z4gaVT5T>)Cc1>8&+P1OK=HMNzfney4O`Ep2cZIqFt-*G(in(bd6v^g!h?udr%1XKy
zia&t1V$pCeEI#T~)$-3ZOFsK7uQI;+G+{iLp$NJj%Ea!M4orUwS}u-Lj)skoL8rVT
zUG7!+m!ihErZe=j=}=!d6-~xisN)V;iY8K_d^UzoP4}HbE51%O=|_`r6CbUHb20KF
zmxzQ05-=6&$!1M%A$Gh={EOC5sB6ncaq5^iySE>6c~E=~%W0o1Vd)dg3@~pB>j5v8
zBVN8}s82)%6lm?hGN707x<1VRnK!yO1$zd~VaY^VVx3#Ygir_H!=xZe1IZlo^3Hm5
zu{|i$xIkT+pI&dQPmF}VD0C%fXc$An(2!*q4)-S_XcfbNGB0nyFopmzF9s)#7dRX1
z$@fAJi*k%)!6s7iv@C~vdgyu(HTW7kCd&wwEYjD*LPA*LcccG(Ml(j-S}|8Jl3cII
zzawCtIg)H?x)uyA%%#3u*U0bYJ*C1Z%lp5Cn^7p$m+PN`FsMjDLCyMY!0|dlTDC9O
zN!8-Pl=NTzE>xTBZ3R!yxTL;ZPszVOFe2pXIRy1x>dWU*cLJlPq<+Dsf}`haoJ>7O
zGdRPBEEByK2R*}*`f`1Co9HO*pHq_lOF2pJLmfR6le}EV?H2k9DI=NvH&DhNM1e%5
zzFhAO3Vrz;PPT9E|HDGRRTz}(!h=FzJ_jWI6*$cLPXMF1PCe(5^L){9^NL28g@%!6
zo?fc22z^E9?-p{B$aqTnq)C5J=no2k9~n=bq&nsf<>?0|eYtLJ5f_M@4@xbw{|AJ=
zjQ`a7Rxvcv2ARrb{3P%+3e<n8FW12XiXlP$D^2G1{}q*``f|NoeYK&bNW;R2x&0B~
z^dwDO?o;b(`S&Qy@sj$Iz7Abl6H0x#{&sr-B1t)eo8~3{HY(`ak@|9-F8|(#{JS3V
z`jUFmulJx!{g?W3JwG8Pj8YCFJV|{?KLT2+j|sLR=rf4m3<kN>lk^O*Qhhl;SB3su
z?Ms7ag-z+doR4-2ec6tbr?yHHtse;!;!{;p|65atQIPtQ+E9??Qqg}G2F-TOvb0V$
zV`S%|P>hxuMcFL-wK?>|enWA~SLhq-UXu{mXN3L(rYdv$EuwvMKUXyxir+9PNLEt%
zMn#rOo4Q{-D7d>6MA<C6bq@VEgue0>`ee^E#JR?V(6190&^17B!6C0F`TIxvGxFNw
diXwA=-6)(j88`c1Gl%}-4Tk<slY)t5{{zyTc;)~A

literal 0
HcmV?d00001

diff --git a/7_multisource_custom/multisource.c b/7_multisource_custom/multisource.c
new file mode 100644
index 0000000..c835908
--- /dev/null
+++ b/7_multisource_custom/multisource.c
@@ -0,0 +1,50 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+#include <assert.h>
+#include "myblas.h"
+
+void __enzyme_double(void*, size_t);
+
+double dotabs(struct complex* alpha, struct complex* beta, int n) {
+  struct complex prod = myblas_cdot(alpha, beta, n);
+  __enzyme_double(&prod, sizeof(struct complex));
+  return myblas_cabs(prod);
+}
+
+void __enzyme_autodiff(void*, ...);
+
+int main(int argc, char *argv[]) {
+  int n = 3;
+  if (argc > 1) {
+    n = atoi(argv[1]);
+  }
+
+
+  struct complex *A = (struct complex*)malloc(sizeof(struct complex) * n);
+  assert(A != 0);
+  for(int i=0; i<n; i++)
+   A[i] = (struct complex){(i+1), (i+2)};
+
+  struct complex *grad_A = (struct complex*)malloc(sizeof(struct complex) * n);
+  assert(grad_A != 0);
+  for(int i=0; i<n; i++)
+   grad_A[i] = (struct complex){0,0};
+
+
+
+  struct complex *B = (struct complex*)malloc(sizeof(struct complex) * n);
+  assert(B != 0);
+  for(int i=0; i<n; i++)
+   B[i] = (struct complex){-3-i, 2*i};
+
+  struct complex *grad_B = (struct complex*)malloc(sizeof(struct complex) * n);
+  assert(grad_B != 0);
+  for(int i=0; i<n; i++)
+   grad_B[i] = (struct complex){0,0};
+
+  __enzyme_autodiff((void*)dotabs, A, grad_A, B, grad_B, n);
+  printf("Gradient dotabs(A)[0] = %f\n", grad_A[0].r);
+
+  return 0;
+}
diff --git a/7_multisource_custom/myblas.c b/7_multisource_custom/myblas.c
new file mode 100644
index 0000000..51c81fb
--- /dev/null
+++ b/7_multisource_custom/myblas.c
@@ -0,0 +1,33 @@
+#include "myblas.h"
+
+struct complex myblas_cdot(struct complex* x, struct complex *y, int n) {
+  struct complex sum = { 0, 0 };
+  for (int i=0; i<n; i++) {
+    sum.r += x[i].r * y[i].r - x[i].i * y[i].i;
+    sum.i += x[i].r * y[i].i + x[i].i * y[i].r;
+  }
+  return sum;
+}
+
+double myblas_cabs(struct complex x) {
+    return x.r * x.r + x.i * x.i;
+}
+
+struct TapeAndComplex __enzyme_augmentfwd(void*, ...);
+struct TapeAndDouble __enzyme_augmentfwd2(void*, ...);
+void __enzyme_reverse(void*, ...);
+struct complex __enzyme_reverse2(void*, ...);
+struct TapeAndComplex myblas_cdot_fwd(struct complex* x, struct complex *dx, struct complex *y, struct complex* dy, int n, int dn) {
+   return __enzyme_augmentfwd((void*)myblas_cdot, x, dx, y, dy, n);
+}
+
+void myblas_cdot_rev(struct complex* x, struct complex* dx, struct complex *y, struct complex* dy, int n, int dn, struct complex* diffret, void* tape) {
+   return __enzyme_reverse((void*)myblas_cdot, x, dx, y, dy, enzyme_dup, n, dn, enzyme_byref, sizeof(struct complex), diffret, tape);
+}
+
+struct TapeAndDouble myblas_cabs_fwd(struct complex x) {
+   return __enzyme_augmentfwd2((void*)myblas_cabs, x);
+}
+struct complex myblas_cabs_rev(struct complex x, double diffret, void* tape) {
+   return __enzyme_reverse2((void*)myblas_cabs, x, diffret, tape);
+}
diff --git a/7_multisource_custom/myblas.h b/7_multisource_custom/myblas.h
new file mode 100644
index 0000000..d980fee
--- /dev/null
+++ b/7_multisource_custom/myblas.h
@@ -0,0 +1,29 @@
+extern int enzyme_const, enzyme_dup, enzyme_out;
+extern int enzyme_byref;
+
+struct complex {
+    double r;
+    double i;
+};
+
+struct complex myblas_cdot(struct complex* x, struct complex *y, int n);
+
+double myblas_cabs(struct complex x);
+
+struct TapeAndComplex {
+    void* tape;
+    struct complex res;
+};
+
+struct TapeAndComplex myblas_cdot_fwd(struct complex* x, struct complex* dx, struct complex *y, struct complex* dy, int n, int dn);
+void myblas_cdot_rev(struct complex* x, struct complex* dx, struct complex *y, struct complex* dy, int n, int dn, struct complex* diffret, void* tape);
+
+struct TapeAndDouble {
+    void* tape;
+    double res;
+};
+
+struct TapeAndDouble myblas_cabs_fwd(struct complex x);
+struct complex myblas_cabs_rev(struct complex x, double diffret, void* tape);
+void* __enzyme_register_gradient1[] = { (void*)myblas_cdot, (void*)myblas_cdot_fwd, (void*)myblas_cdot_rev, "byref_6" };
+void* __enzyme_register_gradient2[] = { (void*)myblas_cabs, (void*)myblas_cabs_fwd, (void*)myblas_cabs_rev };
diff --git a/docker/Dockerfile b/docker/Dockerfile
index ba3b8fe..e3508a9 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -5,13 +5,13 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get -y update && apt-get install -y --no-install-recommends curl gnupg lsb-core software-properties-common \
     && curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key|apt-key add - \
     && apt-add-repository "deb http://apt.llvm.org/`lsb_release -c | cut -f2`/ llvm-toolchain-`lsb_release -c | cut -f2`-12 main" \
-    && apt-get install -y --no-install-recommends autoconf cmake ninja-build gcc g++ libtool gfortran llvm-12-dev lld-12 clang-12 libopenmpi-dev openmpi-bin git \
+    && apt-get install -y --no-install-recommends autoconf cmake ninja-build gcc g++ libtool gfortran llvm-12-dev lld-12 clang-12 libclang-12-dev libopenmpi-dev openmpi-bin git \
     && apt-get autoremove -y --purge \
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/*
 
 # Get & install Enzyme
-RUN git clone https://github.com/wsmoses/Enzyme.git \
+RUN git clone https://github.com/wsmoses/Enzyme.git -b byref \
     && cd Enzyme/enzyme \
     && mkdir build && cd build \
     && cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Debug \