user-0 提问: C语言中如何处理10万组级别的数据

回复:
user: user-1
content: 不要说什么栈空间。 楼主的代码别的不看，就那一个数组的大小，数一数，那是百万行百万列，4TB数量级。 我根本就不信楼主的机器有那么大的内存啊，包括虚拟的。你们信么？ ------------------------------------------------------------ 对数据规模我们应保有基本的敬畏。

user: user-0
content: 您好，多谢您的提醒， 1. 我现有的手上数据是10W行，2列，我如果用malloc分配内存空间的话，就需要2x10万x4字节=0.8Mb的空间，对吗?  2. 如果真的需要处理百万行与百万列的话，如果还是用malloc分配，是否只能在物理上扩大内存来解决？有其他途径吗？

user: user-1
content: 0.8MB直接写数组就行。 但是真的那么大就只能一块一块处理了，先研究算法。 那么大的数据全部装进内存既不现实，也没必要。

user: user-2
content: 仅供参考：  #pragma warning(disable:4996 4244) #include <stdio.h> #include <string.h> #include <stdlib.h> #include <assert.h> #define MAXW 1024 char A[MAXW],B[MAXW]; FILE *fb; #define CUTOFF 8            /* testing shows that this is good value */  /* Note: the theoretical number of stack entries required is    no more than 1 + log2(num).  But we switch to insertion    sort for CUTOFF elements or less, so we really only need    1 + log2(num) - log2(CUTOFF) stack entries.  For a CUTOFF    of 8, that means we need no more than 62 stack entries for    64-bit platforms. */ //#define STKSIZ (8*sizeof(void*) - 2) #define STKSIZ   (8*8             - 2)  /*** *swap(a, b, w) - swap two elements * *Purpose: *       swaps the two file elements of size width * *Entry: *       long long int a, b = pointer to two elements to swap *       long long int width = width in bytes of each file element * *Exit: *       returns void * *Exceptions: * *******************************************************************************/  static void swap(     long long int a,     long long int b,     long long int w     ) {     if ( a != b ) {         _fseeki64(fb,a,SEEK_SET);         fread(A,w,1,fb);         _fseeki64(fb,b,SEEK_SET);         fread(B,w,1,fb);         _fseeki64(fb,a,SEEK_SET);         fwrite(B,w,1,fb);         _fseeki64(fb,b,SEEK_SET);         fwrite(A,w,1,fb);     } }  /*** *shortsort(hi, lo, width, comp) - insertion sort for sorting short file * *Purpose: *       sorts the file of elements between lo and hi (inclusive) *       side effects:  sorts in place *       assumes that lo < hi * *Entry: *       long long int lo = pointer to low element to sort *       long long int hi = pointer to high element to sort *       long long int width = width in bytes of each file element *       int (*comp)() = pointer to function returning analog of strcmp for *               strings, but supplied by user for comparing the file elements. *               it accepts 2 pointers to elements, together with a pointer to a context *               (if present). Returns neg if p1<p2, 0 if p1==p2, pos if p1>p2. * *Exit: *       returns void * *Exceptions: * *******************************************************************************/  static void shortsort(     long long int lo,     long long int hi,     long long int width,     int (*comp)(long long int, long long int,long long int)     ) {     long long int p, maxp;      /* Note: in assertions below, i and j are alway inside original bound of        file to sort. */      while (hi > lo) {         /* A[i] <= A[j] for i <= j, j > hi */         maxp = lo;         for (p = lo+width; p <= hi; p += width) {             /* A[i] <= A[maxp] for lo <= i < p */             if (comp(p, maxp, width) > 0) {                 maxp = p;             }             /* A[i] <= A[maxp] for lo <= i <= p */         }          /* A[i] <= A[maxp] for lo <= i <= hi */          swap(maxp, hi, width);          /* A[i] <= A[hi] for i <= hi, so A[i] <= A[j] for i <= j, j >= hi */          hi -= width;          /* A[i] <= A[j] for i <= j, j > hi, loop top condition established */     }     /* A[i] <= A[j] for i <= j, j > lo, which implies A[i] <= A[j] for i < j,        so file is sorted */ }  /*** *fqsort(base, num, width, comp) - quicksort function for sorting file * *Purpose: *   quicksort the file of elements *   side effects:  sorts in place *   maximum file size is number of elements times size of elements, *   but is limited by the file size of the OS and disk free space. * *Entry: *   long long int base = pointer to base of file *   long long int num  = number of elements in the file *   long long int width = width in bytes of each file element *   int (*comp)() = pointer to function returning analog of strcmp for *           strings, but supplied by user for comparing the file elements. *           it accepts 2 pointers to elements. *           Returns neg if p1<p2, 0 if p1==p2, pos if p1>p2. * *Exit: *   returns void * *Exceptions: *   Input parameters are validated. Refer to the validation section of the function. * *******************************************************************************/  void fqsort(     long long int base,     long long int num,     long long int width,     int (*comp)(long long int, long long int, long long int)     ) {     long long int lo, hi;              /* ends of sub-file currently sorting */     long long int mid;                  /* points to middle of subarray */     long long int loguy, higuy;        /* traveling pointers for partition step */     long long int size;                /* size of the sub-file */     long long int lostk[STKSIZ], histk[STKSIZ];     int stkptr;                 /* stack for saving sub-file to be processed */      /* validation section */     assert(base>=0 && num > 0);     assert(width > 0);     assert(comp != NULL);      if (num < 2)         return;                 /* nothing to do */      stkptr = 0;                 /* initialize stack */      lo = base;     hi = base + width * (num-1);        /* initialize limits */      /* this entry point is for pseudo-recursion calling: setting        lo and hi and jumping to here is like recursion, but stkptr is        preserved, locals aren't, so we preserve stuff on the stack */ recurse:      size = (hi - lo) / width + 1;        /* number of el's to sort */      /* below a certain size, it is faster to use a O(n^2) sorting method */     if (size <= CUTOFF) {         shortsort(lo, hi, width, comp);     }     else {         /* First we pick a partitioning element.  The efficiency of the            algorithm demands that we find one that is approximately the median            of the values, but also that we select one fast.  We choose the            median of the first, middle, and last elements, to avoid bad            performance in the face of already sorted data, or data that is made            up of multiple sorted runs appended together.  Testing shows that a            median-of-three algorithm provides better performance than simply            picking the middle element for the latter case. */          mid = lo + (size / 2) * width;      /* find middle element */          /* Sort the first, middle, last elements into order */         if (comp(lo, mid, width) > 0) {             swap(lo, mid, width);         }         if (comp(lo, hi, width) > 0) {             swap(lo, hi, width);         }         if (comp(mid, hi, width) > 0) {             swap(mid, hi, width);         }          /* We now wish to partition the file into three pieces, one consisting            of elements <= partition element, one of elements equal to the            partition element, and one of elements > than it.  This is done            below; comments indicate conditions established at every step. */          loguy = lo;         higuy = hi;          /* Note that higuy decreases and loguy increases on every iteration,            so loop must terminate. */         for (;;) {             /* lo <= loguy < hi, lo < higuy <= hi,                A[i] <= A[mid] for lo <= i <= loguy,                A[i] > A[mid] for higuy <= i < hi,                A[hi] >= A[mid] */              /* The doubled loop is to avoid calling comp(mid,mid,width), since some                existing comparison funcs don't work when passed the same                value for both pointers. */              if (mid > loguy) {                 do  {                     loguy += width;                 } while (loguy < mid && comp(loguy, mid, width) <= 0);             }             if (mid <= loguy) {                 do  {                     loguy += width;                 } while (loguy <= hi && comp(loguy, mid, width) <= 0);             }              /* lo < loguy <= hi+1, A[i] <= A[mid] for lo <= i < loguy,                either loguy > hi or A[loguy] > A[mid] */              do  {                 higuy -= width;             } while (higuy > mid && comp(higuy, mid, width) > 0);              /* lo <= higuy < hi, A[i] > A[mid] for higuy < i < hi,                either higuy == lo or A[higuy] <= A[mid] */              if (higuy < loguy)                 break;              /* if loguy > hi or higuy == lo, then we would have exited, so                A[loguy] > A[mid], A[higuy] <= A[mid],                loguy <= hi, higuy > lo */              swap(loguy, higuy, width);              /* If the partition element was moved, follow it.  Only need                to check for mid == higuy, since before the swap,                A[loguy] > A[mid] implies loguy != mid. */              if (mid == higuy)                 mid = loguy;              /* A[loguy] <= A[mid], A[higuy] > A[mid]; so condition at top                of loop is re-established */         }          /*     A[i] <= A[mid] for lo <= i < loguy,                A[i] > A[mid] for higuy < i < hi,                A[hi] >= A[mid]                higuy < loguy            implying:                higuy == loguy-1                or higuy == hi - 1, loguy == hi + 1, A[hi] == A[mid] */          /* Find adjacent elements equal to the partition element.  The            doubled loop is to avoid calling comp(mid,mid,width), since some            existing comparison funcs don't work when passed the same value            for both pointers. */          higuy += width;         if (mid < higuy) {             do  {                 higuy -= width;             } while (higuy > mid && comp(higuy, mid, width) == 0);         }         if (mid >= higuy) {             do  {                 higuy -= width;             } while (higuy > lo && comp(higuy, mid, width) == 0);         }          /* OK, now we have the following:               higuy < loguy               lo <= higuy <= hi               A[i]  <= A[mid] for lo <= i <= higuy               A[i]  == A[mid] for higuy < i < loguy               A[i]  >  A[mid] for loguy <= i < hi               A[hi] >= A[mid] */          /* We've finished the partition, now we want to sort the subarrays            [lo, higuy] and [loguy, hi].            We do the smaller one first to minimize stack usage.            We only sort arrays of length 2 or more.*/          if ( higuy - lo >= hi - loguy ) {             if (lo < higuy) {                 lostk[stkptr] = lo;                 histk[stkptr] = higuy;                 ++stkptr;             }                           /* save big recursion for later */              if (loguy < hi) {                 lo = loguy;                 goto recurse;           /* do small recursion */             }         }         else {             if (loguy < hi) {                 lostk[stkptr] = loguy;                 histk[stkptr] = hi;                 ++stkptr;               /* save big recursion for later */             }              if (lo < higuy) {                 hi = higuy;                 goto recurse;           /* do small recursion */             }         }     }      /* We have sorted the file, except for any pending sorts on the stack.        Check if there are any, and do them. */      --stkptr;     if (stkptr >= 0) {         lo = lostk[stkptr];         hi = histk[stkptr];         goto recurse;           /* pop subarray from stack */     }     else         return;                 /* all subarrays done */ }   /*** *long long int fbsearch() - do a binary search on an file * *Purpose: *   Does a binary search of a sorted file for a key. * *Entry: *   const void *key    - key to search for *   const long long int base   - base of sorted file to search *   unsigned int num   - number of elements in file *   unsigned int width - number of bytes per element *   int (*compare)()   - pointer to function that compares key and file *           elements, returning neg when #1 < #2, pos when #1 > #2, and *           0 when they are equal. Function is passed pointers to key *           and pointer to file elements. * *Exit: *   if key is found: *           returns pointer to occurrence of key in file *   if key is not found: *           returns -1 * *Exceptions: *   Input parameters are validated. Refer to the validation section of the function. * *******************************************************************************/  long long int fbsearch(     const void *key,     long long int base,     long long int num,     long long int width,     int (*compare)(const void *, long long int, long long int)     ) {     long long int lo = (long long int)base;     long long int hi = (long long int)base + (num - 1) * width;     long long int mid;     long long int half;     int result;      /* validation section */     assert(base>=0 && num > 0);     assert(width > 0);     assert(compare != NULL);          /*         We allow a NULL key here because it breaks some older code and because we do not dereference         this ourselves so we can't be sure that it's a problem for the comparison function         */      while (lo <= hi)     {         if ((half = num / 2) != 0)         {             mid = lo + (num & 1 ? half : (half - 1)) * width;             if (!(result = compare(key, mid, width)))                 return(mid);             else if (result < 0)             {                 hi = mid - width;                 num = num & 1 ? half : half-1;             }             else             {                 lo = mid + width;                 num = half;             }         }         else if (num)             return (compare(key, lo, width) ? -1 : lo);         else             break;     }      return -1; } int cmp(long long int a, long long int b, long long int w) {     _fseeki64(fb,a,SEEK_SET);     fread(A,w,1,fb);     _fseeki64(fb,b,SEEK_SET);     fread(B,w,1,fb);     return strncmp(A,B,w); } int cmps(const void *k, long long int b, long long int w) {     _fseeki64(fb,b,SEEK_SET);     fread(B,w,1,fb);     return strncmp((const char *)k,B,w); } void test(int n,int w,void *k1,void *k2) {     int i,v;     long long int r;      assert(w<MAXW);     fb=fopen("test.bin","wb");     if (NULL==fb) {         printf("Can not create file test.bin!\n");         return;     }     for (i=0;i<n;i++) {         v=10+rand()%90;         printf("%02d ",v);         fprintf(fb,"%02d\r\n",v);     }     printf("\n");     fclose(fb);      fb=fopen("test.bin","rb+");     if (NULL==fb) {         printf("Can not open file test.bin!\n");         return;     }     fqsort(0,n,w,cmp);     r=fbsearch(k1,0,n,w,cmps);     printf("%s r1:%lld\n",k1,r);     r=fbsearch(k2,0,n,w,cmps);     printf("%s r2:%lld\n",k2,r);     fclose(fb);     system("type test.bin"); } int main() {     srand(0);      test(  8,4,"75\r\n","12\r\n");     test(100,4,"94\r\n","95\r\n");      return 0; }

user: user-3
content: 栈没那么大，前面加上static试试，如果不行，建议把数组定义为合适的大小，数组定义这么大，是可以存放10w × 10w个float数据的，这个是你需要的大小吗？

user: user-3
content: 即使加上static不合适，那么把数组定义成合适大小，也需要加上static

user: user-4
content: 使用文件游标,一次读一部分数据(固定行数,比如100行)到缓冲区处理.

user: user-5
content: 把数组a做成全局变量，或者用动态内存分配。