[PATCH 4/4] oleaut32: handle by-val to by-reference passing magic on x86_64

The Win64 API does not pass structures on the stack, but implicit
by reference. Handle this during the generated stub/proxy calls.

Also changed argsize to always return 1 on x86_64 due to this.

Fixes https://bugs.winehq.org/show_bug.cgi?id=26768

Signed-off-by: Marcus Meissner <[email protected]>
---
 dlls/oleaut32/tmarshal.c | 103 ++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 84 insertions(+), 19 deletions(-)

diff --git a/dlls/oleaut32/tmarshal.c b/dlls/oleaut32/tmarshal.c
index e71e9771f4..f269822f08 100644
--- a/dlls/oleaut32/tmarshal.c
+++ b/dlls/oleaut32/tmarshal.c
@@ -135,16 +135,16 @@ _unmarshal_interface(marshal_state *buf, REFIID riid, LPUNKNOWN *pUnk) {
     DWORD		xsize;
 
     TRACE("...%s...\n",debugstr_guid(riid));
-    
+
     *pUnk = NULL;
     hres = xbuf_get(buf,(LPBYTE)&xsize,sizeof(xsize));
     if (hres) {
         ERR("xbuf_get failed\n");
         return hres;
     }
-    
+
     if (xsize == 0) return S_OK;
-    
+
     hres = CreateStreamOnHGlobal(0,TRUE,&pStm);
     if (hres) {
 	ERR("Stream create failed %x\n",hres);
@@ -659,8 +659,10 @@ static const IRpcProxyBufferVtbl tmproxyvtable = {
 
 /* how much space do we use on stack in DWORD_PTR steps. */
 static int
-_argsize(TYPEDESC *tdesc, ITypeInfo *tinfo) {
+_argsize(TYPEDESC *tdesc, ITypeInfo *tinfo, int *byref) {
     DWORD ret;
+
+    *byref = 0;
     switch (tdesc->vt) {
     case VT_I8:
     case VT_UI8:
@@ -677,9 +679,11 @@ _argsize(TYPEDESC *tdesc, ITypeInfo *tinfo) {
         break;
     case VT_DECIMAL:
         ret = sizeof(DECIMAL);
+	*byref = 1;
         break;
     case VT_VARIANT:
         ret = sizeof(VARIANT);
+	*byref = 1;
         break;
     case VT_USERDEFINED:
     {
@@ -687,6 +691,7 @@ _argsize(TYPEDESC *tdesc, ITypeInfo *tinfo) {
         TYPEATTR *tattr;
         HRESULT hres;
 
+	*byref = 1;
         hres = ITypeInfo_GetRefTypeInfo(tinfo,tdesc->u.hreftype,&tinfo2);
         if (FAILED(hres))
             return 0; /* should fail critically in serialize_param */
@@ -701,6 +706,12 @@ _argsize(TYPEDESC *tdesc, ITypeInfo *tinfo) {
         break;
     }
 
+#ifdef __x86_64__
+    /* On x86_64 we do not store byvalue things on the stack, everything
+     * complex or larger than a register will be spilled and passed by reference */
+    if (*byref)
+        return 1;
+#endif
     return (ret + sizeof(DWORD_PTR) - 1) / sizeof(DWORD_PTR);
 }
 
@@ -749,6 +760,9 @@ _xsize(const TYPEDESC *td, ITypeInfo *tinfo) {
         ITypeInfo_Release(tinfo2);
         return ret;
     }
+    case VT_UINT:
+    case VT_INT:
+        return 4;
     default:
 	return sizeof(DWORD_PTR);
     }
@@ -888,6 +902,7 @@ serialize_param(
                 }
                 break;
 	    case TKIND_ENUM:	/* confirmed */
+		break;
 	    case TKIND_RECORD:	/* FIXME: mostly untested */
 		break;
 	    case TKIND_DISPATCH:	/* will be done in VT_USERDEFINED case */
@@ -1458,7 +1473,7 @@ static DWORD WINAPI xCall(int method, void **args)
     UINT		nrofnames;
     DWORD		remoteresult = 0;
     ITypeInfo 		*tinfo;
-    IRpcChannelBuffer *chanbuf;
+    IRpcChannelBuffer	*chanbuf;
 
     EnterCriticalSection(&tpinfo->crit);
 
@@ -1509,17 +1524,21 @@ static DWORD WINAPI xCall(int method, void **args)
     xargs = (DWORD_PTR *)(args + 1);
     for (i=0;i<fdesc->cParams;i++) {
 	ELEMDESC	*elem = fdesc->lprgelemdescParam+i;
+	DWORD_PTR	*argsptr;
+	int		byref, argsize;
+
 	if (TRACE_ON(olerelay)) {
 	    if (i) TRACE_(olerelay)(",");
 	    if (i+1<nrofnames && names[i+1])
 		TRACE_(olerelay)("%s=",relaystr(names[i+1]));
 	}
+        argsize = _argsize(&elem->tdesc, tinfo, &byref);
 	/* No need to marshal other data than FIN and any VT_PTR. */
         if (!is_in_elem(elem))
         {
             if (elem->tdesc.vt != VT_PTR)
             {
-                xargs+=_argsize(&elem->tdesc, tinfo);
+                xargs += argsize;
                 TRACE_(olerelay)("[out]");
                 continue;
             }
@@ -1529,13 +1548,17 @@ static DWORD WINAPI xCall(int method, void **args)
             }
         }
 
+	argsptr = xargs;
+	if (byref)
+	    argsptr = *(DWORD_PTR**)xargs;
+
 	hres = serialize_param(
 	    tinfo,
 	    is_in_elem(elem),
 	    TRACE_ON(olerelay),
 	    FALSE,
 	    &elem->tdesc,
-	    xargs,
+	    argsptr,
 	    &buf
 	);
 
@@ -1543,7 +1566,7 @@ static DWORD WINAPI xCall(int method, void **args)
 	    ERR("Failed to serialize param, hres %x\n",hres);
 	    break;
 	}
-	xargs+=_argsize(&elem->tdesc, tinfo);
+	xargs += argsize;
     }
     TRACE_(olerelay)(")");
 
@@ -1577,23 +1600,31 @@ static DWORD WINAPI xCall(int method, void **args)
     status = S_OK;
     for (i=0;i<fdesc->cParams;i++) {
 	ELEMDESC	*elem = fdesc->lprgelemdescParam+i;
+	DWORD_PTR	*argsptr;
+	int		argsize, byref;
 
         if (i) TRACE_(olerelay)(",");
         if (i+1<nrofnames && names[i+1]) TRACE_(olerelay)("%s=",relaystr(names[i+1]));
 
+	argsize = _argsize(&elem->tdesc, tinfo, &byref);
 	/* No need to marshal other data than FOUT and any VT_PTR */
 	if (!is_out_elem(elem) && (elem->tdesc.vt != VT_PTR)) {
-	    xargs += _argsize(&elem->tdesc, tinfo);
+	    xargs += argsize;
 	    TRACE_(olerelay)("[in]");
 	    continue;
 	}
+
+	argsptr = xargs;
+	if (byref)
+            argsptr = *(DWORD_PTR**)xargs;
+
 	hres = deserialize_param(
 	    tinfo,
 	    is_out_elem(elem),
 	    TRACE_ON(olerelay),
 	    FALSE,
 	    &(elem->tdesc),
-	    xargs,
+	    argsptr,
 	    &buf
         );
 	if (hres) {
@@ -1601,7 +1632,7 @@ static DWORD WINAPI xCall(int method, void **args)
 	    status = hres;
 	    break;
 	}
-	xargs += _argsize(&elem->tdesc, tinfo);
+	xargs += argsize;
     }
 
     hres = xbuf_get(&buf, (LPBYTE)&remoteresult, sizeof(DWORD));
@@ -1832,7 +1863,7 @@ static HRESULT init_proxy_entry_point(TMProxyImpl *proxy, unsigned int num)
 {
     int j;
     /* nrofargs including This */
-    int nrofargs = 1;
+    int nrofargs = 1, byref;
     ITypeInfo *tinfo2;
     TMAsmProxy	*xasm = proxy->asmstubs + num;
     HRESULT hres;
@@ -1846,7 +1877,7 @@ static HRESULT init_proxy_entry_point(TMProxyImpl *proxy, unsigned int num)
     ITypeInfo_Release(tinfo2);
     /* some args take more than 4 byte on the stack */
     for (j=0;j<fdesc->cParams;j++)
-        nrofargs += _argsize(&fdesc->lprgelemdescParam[j].tdesc, proxy->tinfo);
+        nrofargs += _argsize(&fdesc->lprgelemdescParam[j].tdesc, proxy->tinfo, &byref);
 
 #ifdef __i386__
     if (fdesc->callconv != CC_STDCALL) {
@@ -1872,6 +1903,21 @@ static HRESULT init_proxy_entry_point(TMProxyImpl *proxy, unsigned int num)
 
 #elif defined(__x86_64__)
 
+/* Stack layout:
+ * We store the 4 register arguments in the supplied "register save zone"
+ * so all arguments line up on the stack, same as in i386 mode.
+ * ... further args ...
+ * arg6
+ * arg5
+ * zone0	8	+0x28 arg4
+ * zone0	8	+0x20 arg3
+ * zone0	8	+0x18 arg2
+ * zone0	8	+0x10 arg1	< %RDX (arg2) before call
+ * return RIP	8
+ * saved RBP	8	<= RBP in assembler stub
+ * ..
+ */
+
     xasm->pushq_rbp         = 0x55;         /* pushq %rbp */
     xasm->movq_rsp_rbp[0]   = 0x48;         /* movq %rsp,%rbp */
     xasm->movq_rsp_rbp[1]   = 0x89;
@@ -2124,7 +2170,7 @@ TMStubImpl_Invoke(
     LPRPCSTUBBUFFER iface, RPCOLEMESSAGE* xmsg,IRpcChannelBuffer*rpcchanbuf)
 {
 #if defined(__i386__) || defined(__x86_64__)
-    int		i;
+    int		i, byref;
     const FUNCDESC *fdesc;
     TMStubImpl *This = impl_from_IRpcStubBuffer(iface);
     HRESULT	hres;
@@ -2193,7 +2239,7 @@ TMStubImpl_Invoke(
     /*dump_FUNCDESC(fdesc);*/
     nrofargs = 0;
     for (i=0;i<fdesc->cParams;i++)
-	nrofargs += _argsize(&fdesc->lprgelemdescParam[i].tdesc, tinfo);
+	nrofargs += _argsize(&fdesc->lprgelemdescParam[i].tdesc, tinfo, &byref);
     args = HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, (nrofargs+1)*sizeof(DWORD_PTR));
     if (!args)
     {
@@ -2205,6 +2251,15 @@ TMStubImpl_Invoke(
     xargs = args+1;
     for (i=0;i<fdesc->cParams;i++) {
 	ELEMDESC	*elem = fdesc->lprgelemdescParam+i;
+	DWORD_PTR	*argsptr;
+	int		argsize, byref;
+
+	argsize = _argsize(&elem->tdesc, tinfo, &byref);
+	argsptr = xargs;
+	if (byref) {
+            *xargs = (DWORD_PTR)HeapAlloc(GetProcessHeap(),HEAP_ZERO_MEMORY,_xsize(&elem->tdesc, tinfo));
+            argsptr = *(DWORD_PTR**)xargs;
+        }
 
 	hres = deserialize_param(
 	   tinfo,
@@ -2212,10 +2267,10 @@ TMStubImpl_Invoke(
 	   FALSE,
 	   TRUE,
 	   &(elem->tdesc),
-	   xargs,
+	   argsptr,
 	   &buf
 	);
-	xargs += _argsize(&elem->tdesc, tinfo);
+	xargs += argsize;
 	if (hres) {
 	    ERR("Failed to deserialize param %s, hres %x\n",relaystr(names[i+1]),hres);
 	    break;
@@ -2252,16 +2307,26 @@ TMStubImpl_Invoke(
     xargs = args+1;
     for (i=0;i<fdesc->cParams;i++) {
 	ELEMDESC	*elem = fdesc->lprgelemdescParam+i;
+	DWORD_PTR	*argsptr;
+	int		argsize, byref;
+
+	argsize = _argsize(&elem->tdesc, tinfo, &byref);
+	argsptr = xargs;
+	if (byref)
+            argsptr = *(DWORD_PTR**)xargs;
 	hres = serialize_param(
 	   tinfo,
 	   is_out_elem(elem),
 	   FALSE,
 	   TRUE,
 	   &elem->tdesc,
-	   xargs,
+	   argsptr,
 	   &buf
 	);
-	xargs += _argsize(&elem->tdesc, tinfo);
+	if (byref)
+            HeapFree(GetProcessHeap(),HEAP_ZERO_MEMORY,*(DWORD_PTR**)argsptr);
+
+	xargs += argsize;
 	if (hres) {
 	    ERR("Failed to stuballoc param, hres %x\n",hres);
 	    break;
-- 
2.14.3