Add XNFcallocarray() to allow xnfcalloc() to check for overflow

The xnfcalloc() macro took two arguments but simply multiplied them
together without checking for overflow and defeating any overflow
checking that calloc() might have done.  Let's not do that.

The original XNFcalloc() function is left for now to preserve driver
ABI, but is marked as deprecated so it can be removed in a future round
of ABI break/cleanup.

Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>
Reviewed-by: Matt Turner <mattst88@gmail.com>

config/config.c

@@ -130,7 +130,8 @@ device_is_duplicate(const char *config_info)
 struct OdevAttributes *
 config_odev_allocate_attributes(void)
 {
-    struct OdevAttributes *attribs = XNFcalloc(sizeof (struct OdevAttributes));
+    struct OdevAttributes *attribs =
+        xnfcalloc(1, sizeof (struct OdevAttributes));
     attribs->fd = -1;
     return attribs;
 }

hw/xfree86/utils/cvt/cvt.c

@@ -54,11 +54,11 @@ XNFalloc(unsigned long n)
 
 /* xnfcalloc implementation used by the server code we built in */
 void *
-XNFcalloc(unsigned long n)
+XNFcallocarray(size_t nmemb, size_t size)
 {
     void *r;
 
-    r = calloc(1, n);
+    r = calloc(nmemb, size);
     if (!r) {
         perror("calloc failed");
         exit(1);

include/os.h

@@ -69,7 +69,7 @@ typedef struct _NewClientRec *NewClientPtr;
 
 #ifndef xnfalloc
 #define xnfalloc(size) XNFalloc((unsigned long)(size))
-#define xnfcalloc(_num, _size) XNFcalloc((unsigned long)(_num)*(unsigned long)(_size))
+#define xnfcalloc(_num, _size) XNFcallocarray((_num), (_size))
 #define xnfrealloc(ptr, size) XNFrealloc((void *)(ptr), (unsigned long)(size))
 
 #define xstrdup(s) Xstrdup(s)
@@ -222,7 +222,14 @@ XNFalloc(unsigned long /*amount */ );
  * enough memory.
  */
 extern _X_EXPORT void *
-XNFcalloc(unsigned long /*amount */ );
+XNFcalloc(unsigned long /*amount */ ) _X_DEPRECATED;
+
+/*
+ * This function calloc(3)s buffer, terminating the server if there is not
+ * enough memory or the arguments overflow when multiplied
+ */
+extern _X_EXPORT void *
+XNFcallocarray(size_t nmemb, size_t size);
 
 /*
  * This function realloc(3)s passed buffer, terminating the server if there is

os/utils.c

@@ -1128,10 +1128,20 @@ XNFalloc(unsigned long amount)
     return ptr;
 }
 
+/* The original XNFcalloc was used with the xnfcalloc macro which multiplied
+ * the arguments at the call site without allowing calloc to check for overflow.
+ * XNFcallocarray was added to fix that without breaking ABI.
+ */
 void *
 XNFcalloc(unsigned long amount)
 {
-    void *ret = calloc(1, amount);
+    return XNFcallocarray(1, amount);
+}
+
+void *
+XNFcallocarray(size_t nmemb, size_t size)
+{
+    void *ret = calloc(nmemb, size);
 
     if (!ret)
         FatalError("XNFcalloc: Out of memory");