Issue 1:
by goprogger@gmx.de:

Before filing a bug, please check whether it has been fixed since
the latest release: run "hg pull -u" and retry what you did to
reproduce the problem. Thanks.

What steps will reproduce the problem?
package main

Version 1.go:
-------------

import (
"fmt"
"runtime"
)

func main() {
F(15000000)
}

func F(n int) (r string) {
r2 := ""
for len(r2 + r) < n {
t := ""

for i := 0; i < 5000; i++ {
t += "10"
}
r += t

if len(r) > 1000000 {
fmt.Printf("Alloc: %12d\n", runtime.MemStats.Alloc)
r2 += r
r = ""
}
}

r2 += r
return r2[:n]
}

Version 0.go (crushes ever faster):
-----------------------------------

package main

import (
"fmt"
"runtime"
"crypto/rand"
)

func main() {
F(15000000)

b := make([]byte, 1000)
_, err := rand.Read(b)
if err != nil {
return
}
}

func F(n int) (r string) {
r2 := ""
for len(r2 + r) < n {
t := ""

for i := 0; i < 5000; i++ {
t += "10"
}
r += t

if len(r) > 1000000 {
fmt.Printf("Alloc: %10d\n", runtime.MemStats.Alloc)
r2 += r
r = ""
}
}

r2 += r
return r2[:n]
}

What is the expected output?
Output of 0.go compiled with 6g

Alloc: 2730712
Alloc: 4285864
Alloc: 6966696
Alloc: 6770088
Alloc: 13104552
Alloc: 13534632
Alloc: 17578920
Alloc: 19774376
Alloc: 18771368
Alloc: 17745320
Alloc: 16735656
Alloc: 16569768
Alloc: 58090368
Alloc: 60248960

What do you see instead?

Output 386 0.go

Alloc: 4590192
Alloc: 18337264
Alloc: 24819696
Alloc: 354659856
runtime: memory allocated by OS not in usable range
runtime: out of memory: cannot allocate 4325376-byte block (535953408 in use)
throw: out of memory

runtime.throw+0x43 /home/ubuntu/go/src/pkg/runtime/runtime.c:102
runtime.throw(0x80f3005, 0x413)
runtime.mallocgc+0x2cd /home/ubuntu/go/src/pkg/runtime/malloc.c:60
runtime.mallocgc(0x4127b1, 0x0, 0x1, 0x1, 0x0, ...)
runtime.mal+0x43 /home/ubuntu/go/src/pkg/runtime/malloc.c:289
runtime.mal(0x4127b1, 0x805bfb1)
runtime.gostringsize+0x49 /home/ubuntu/go/src/pkg/runtime/string.c:40
runtime.gostringsize(0x80fe4c, 0x4127b0, 0x2710)
concatstring+0x79 /home/ubuntu/go/src/pkg/runtime/string.c:126
concatstring(0x80fe88, 0x2, 0x80fe78, 0x80fe78)
runtime.concatstring+0x1f /home/ubuntu/go/src/pkg/runtime/string.c:141
runtime.concatstring(0x2, 0xaf067000, 0x3da540)
main.F+0x6a /home/ubuntu/test/0.go:21
main.F(0xe4e1c0, 0xb6c33000, 0x38270)
main.main+0x26 /home/ubuntu/test/0.go:10
main.main()
runtime.mainstart+0xf /home/ubuntu/go/src/pkg/runtime/386/asm.s:93
runtime.mainstart()
runtime.goexit /home/ubuntu/go/src/pkg/runtime/proc.c:178
runtime.goexit()
----- goroutine created by -----
_rt0_386+0xc1 /home/ubuntu/go/src/pkg/runtime/386/asm.s:80

--------------------------------------------------------------------------
Output 386 1.go

Alloc: 3840488
Alloc: 4666216
Alloc: 8051560
Alloc: 6943592
Alloc: 8584040
Alloc: 13527912
Alloc: 20775784
Alloc: 91599424
Alloc: 424534984
runtime: memory allocated by OS not in usable range
runtime: out of memory: cannot allocate 9175040-byte block (536805376 in use)
throw: out of memory

runtime.throw+0x43 /home/ubuntu/go/src/pkg/runtime/runtime.c:102
runtime.throw(0x80e9005, 0x8b1)
runtime.mallocgc+0x2cd /home/ubuntu/go/src/pkg/runtime/malloc.c:60
runtime.mallocgc(0x8b01f1, 0x0, 0x1, 0x1, 0x0, ...)
runtime.mal+0x43 /home/ubuntu/go/src/pkg/runtime/malloc.c:289
runtime.mal(0x8b01f1, 0x805bf4a)
runtime.gostringsize+0x49 /home/ubuntu/go/src/pkg/runtime/string.c:40
runtime.gostringsize(0x307e8c, 0x8b01f0, 0x2710)
concatstring+0x79 /home/ubuntu/go/src/pkg/runtime/string.c:126
concatstring(0x307ec8, 0x2, 0x307eb8, 0x307eb8)
runtime.concatstring+0x1f /home/ubuntu/go/src/pkg/runtime/string.c:141
runtime.concatstring(0x2, 0xb5d6d000, 0x8ab3d0)
main.F+0x6a /home/ubuntu/test/1.go:14
main.F(0xe4e1c0, 0xb451f000, 0x4e20)
main.main+0x26 /home/ubuntu/test/1.go:9
main.main()
runtime.mainstart+0xf /home/ubuntu/go/src/pkg/runtime/386/asm.s:93
runtime.mainstart()
runtime.goexit /home/ubuntu/go/src/pkg/runtime/proc.c:178
runtime.goexit()
----- goroutine created by -----
_rt0_386+0xc1 /home/ubuntu/go/src/pkg/runtime/386/asm.s:80

Which compiler are you using (5g, 6g, 8g, gccgo)?
8g

Which operating system are you using?
ubuntu 10.04, 10.10, 11.04

Which revision are you using? (hg identify)
e1e194eb5c8e (release-branch.r57)

Please provide any additional information below.

Issue 2:
by serge.hulne:

Problem:

The following code uses a map to store unique bigrams (couple of
words) from a text file (and an array to sort them according to their
frequency of occurence).

The file used for the test is :
http://www.gutenberg.org/cache/epub/100/pg100.txt
(concatenated a couple of times for the sake of the test)

At no point during the run of program does the map store an amount of
data comparable to the size of the text file, since the most frequent
bigrams appear thousands of time in said text file.

One would therefore expect the memory usage of the executable to use
an amount of RAM which is less than the size of the text file.

However it seem to be quite the opposite in practice.
Here is the source code:

The version of 6g is : 6g version 5189 release.2010-03-30 release
The hardware is : Mac Mini , Intel Core Duo

The Go env. variables are:
GOBIN=/Users/serge2/go/src/bin
GOARCH=amd64
GOROOT=/Users/serge2/go
GOOS=darwin

Serge Hulne

-------------------
package main

import (
"fmt"
"os"
"bufio"
"unicode"
"strings"
"time"
"runtime"
"sort"
)

func monitor() {
for {
fmt.Printf("Memory usage = %d\n",
runtime.MemStats.Alloc)
time.Sleep(1e9)
}

}

//---
type Bigram struct {
key string
value string
freq uint

}

type BigramArray []Bigram

// Methods required by sort.Interface to sort structures of the type
BigramArray.
func (s BigramArray) Len() int { return len(s) }
func (s BigramArray) Less(i, j int) bool { return s[i].freq >
s[j].freq } //(reverse sort)
func (s BigramArray) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

func hash_bigram(b *Bigram) uint {
var h uint = 5381
for _, r := range (*b).key {
h = (h << 5) + h + uint(r)
}
h = (h << 5) + h + uint('_')
for _, r := range (*b).value {
h = (h << 5) + h + uint(r)
}
return h
}

//---

func main() int {

l_cnt := 0
w_cnt := 0
cpt_chars := 0
inword := false
buf := ""
old_word := ""
new_word := ""
var bigram Bigram

T := map [uint] Bigram{}

//f, err := os.Open("faust.txt", os.O_RDONLY, 0666)
//f, err := os.Open("test.txt", os.O_RDONLY, 0666)
//f, err := os.Open("hamlet.txt", os.O_RDONLY, 0666)
f, err := os.Open("../shakespeare.txt", os.O_RDONLY, 0666)
//f, err := os.Open("/Users/serge2/development/NLP/B60W20.txt",
os.O_RDONLY, 0666)
//f, err := os.Open("../B60W20.txt", os.O_RDONLY, 0666)

go monitor()

if err != nil {
fmt.Printf("\nError => %s\n\n", err)
}

reader := bufio.NewReader(f) //Buffered reader

for {
c, _ ,err := reader.ReadRune() //"ReadRune": reads unicode
chars
cpt_chars++
if err != os.EOF && err == nil {
if c == '\n' {
l_cnt++
}
///////////////////////////////
if unicode.IsSpace(c) == false { //Scanner :
filters out whitespace
if inword == false {
buf = ""
buf += string(c)
inword = true
w_cnt++
} else {
buf += string(c)
}
} else if inword == true {

//---
buf = strings.ToLower(buf)
new_word = buf
if len(new_word)>0 && len(old_word)>0 {
bigram.key = old_word
bigram.value = new_word
h := hash_bigram(&bigram)
if _, ok := T[h]; ok {
b := T[h]
b.freq++
T[h] = b
} else {
T[h] = bigram
}
}
old_word = new_word
//---

//fmt.Printf("%d\t buf = (%s)\n", w_cnt ,buf)
inword = false
buf = ""
}
///////////////////////////////
} else { //EOF detected
if err == os.EOF {
break
}
} //end if (err=nil)
} // end for (main loop)

//---
var bigramsArray BigramArray
bigramsArray = make (BigramArray, len(T))

i := 0
for _, v := range T{
bigramsArray[i] = v
i++
}

sort.Sort(bigramsArray)

for _, v := range bigramsArray[0:20]{
fmt.Printf("%d\t (%s,%s)\n", v.freq, v.key, v.value)
}
//---

fmt.Printf("\nlines = %d, words = %d, chars = %d\n", l_cnt, w_cnt,
cpt_chars)
return 0

}

-------------------

Here is the result of the run:

serge-hulnes-mac-mini:Go_test serge2$ ls -l ../shakespeare.txt
-rw-r--r-- 1 serge2 staff 15862370 5 mar 11:07 ../shakespeare.txt
serge-hulnes-mac-mini:Go_test serge2$ time ./bigrammes_7
Memory usage = 347648
Memory usage = 18181624
Memory usage = 33399512
Memory usage = 22003136
Memory usage = 32615280
Memory usage = 19154104
Memory usage = 29662040
Memory usage = 18441376
Memory usage = 28763680
Memory usage = 18525536
Memory usage = 28688216
Memory usage = 18481616
Memory usage = 28727720
6809 (of,the)
5619 (in,the)
5539 (to,the)
4699 (i,am)
4149 (i,will)
3459 (i,have)
3079 (it,is)
2769 (if,you)
2609 (that,i)
2589 (and,the)
2569 (by,the)
2469 (to,be)
2099 (and,i)
2059 (my,lord,)
2039 (of,syracuse.)
2019 (you,are)
1999 (is,the)
1999 (for,the)
1989 (antipholus,of)
1899 (of,my)

lines = 495200, words = 2703240, chars = 15862351

real 0m12.725s
user 0m12.602s
sys 0m0.104s

Attachments:

bigrammes_7.go (2930 bytes)