helloadhavan/github_issues huggingface dataset:
Repository: huggingface/transformers
Pull request: [`Tokenizers`] Change treatment of special tokens

Related to #39585 where we try to make some previously made special tokens to non-special tokens 

Changed files:
filename: src/transformers/tokenization_utils_base.py
patch: @@ -2152,9 +2152,10 @@ def save_pretrained(
         # Add tokenizer class to the tokenizer config to be able to reload it with from_pretrained
         tokenizer_class = self.__class__.__name__
 
-        # tokenizers backend don't need to save added_tokens_decoder
+        # tokenizers backend don't need to save added_tokens_decoder and additional_special_tokens
         if any(base.__name__ == "TokenizersBackend" for base in self.__class__.__mro__):
             tokenizer_config.pop("added_tokens_decoder", None)
+            tokenizer_config.pop("additional_special_tokens", None)
 
         # Remove the Fast at the end if we can save the slow tokenizer
         if tokenizer_class.endswith("Fast") and getattr(self, "can_save_slow_tokenizer", False):

filename: src/transformers/tokenization_utils_tokenizers.py
patch: @@ -339,7 +339,7 @@ def __init__(self, *args, **kwargs):
                 tokens.append(token)
             if tokens:
                 # These tokens are from the special tokens map
-                self.add_tokens(tokens, special_tokens=True)
+                self.add_tokens(tokens)
 
         try:
             vocab_size = self._tokenizer.get_vocab_size()

filename: tests/tokenization/test_tokenization_utils.py
patch: @@ -44,7 +44,7 @@
 
 if is_tokenizers_available():
     import tokenizers
-    from tokenizers import Tokenizer
+    from tokenizers import AddedToken, Tokenizer
     from tokenizers.models import WordPiece
 
 
@@ -330,3 +330,23 @@ def test_encode_message_raises_on_add_generation_prompt(self):
         ]
         with self.assertRaises(ValueError):
             tokenizer.encode_message_with_chat_template(conversation[0], add_generation_prompt=True)
+
+    @require_tokenizers
+    def test_special_tokens_overwrite(self):
+        text_with_nonspecial_tokens = "there are 2 cats"  # '2' is originally special
+
+        tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/Ernie4_5_Tokenizer")
+        # Overwrite special tokens 0-9 to non-special
+        tokenizer.add_tokens([AddedToken(f"{i}", normalized=False, special=False) for i in range(10)])
+        self.assertTrue(
+            tokenizer.decode(tokenizer.encode(text_with_nonspecial_tokens), skip_special_tokens=True)
+            == text_with_nonspecial_tokens
+        )
+
+        # Checking if this carries over even after saving and relaoding
+        tokenizer.save_pretrained("/tmp/ernie_tokenizer")
+        new_tokenizer = AutoTokenizer.from_pretrained("/tmp/ernie_tokenizer")
+        self.assertTrue(
+            new_tokenizer.decode(new_tokenizer.encode(text_with_nonspecial_tokens), skip_special_tokens=True)
+            == text_with_nonspecial_tokens
+        )
Repository: facebook/react
Pull request: [eprh] Bump stable version


https://www.npmjs.com/package/eslint-plugin-react-hooks/v/6.0.0 was just released, so we can bump this now.
---
[//]: # (BEGIN SAPLING FOOTER)
Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/facebook/react/pull/32978).
* #32979
* __->__ #32978

Changed files:
filename: ReactVersions.js
patch: @@ -33,7 +33,7 @@ const canaryChannelLabel = 'canary';
 const rcNumber = 0;
 
 const stablePackages = {
-  'eslint-plugin-react-hooks': '6.0.0',
+  'eslint-plugin-react-hooks': '6.1.0',
   'jest-react': '0.17.0',
   react: ReactVersion,
   'react-art': ReactVersion,
Repository: ollama/ollama
Pull request: readme + examples: add pgai to readme projects and add example for semantic search

Hey there,
adding a mention of pgai to the community integrations section in the Readme as well as a small demo script for the example folder.
Let me know if you'd like to change anything!

Changed files:
filename: README.md
patch: @@ -407,6 +407,7 @@ See the [API documentation](./docs/api.md) for all endpoints.
 
 ### Database
 
+- [PostgreSQL extension pgai](https://github.com/timescale/pgai) (Create and search embeddings from Ollama models using pgvector)
 - [MindsDB](https://github.com/mindsdb/mindsdb/blob/staging/mindsdb/integrations/handlers/ollama_handler/README.md) (Connects Ollama models with nearly 200 data platforms and apps)
 - [chromem-go](https://github.com/philippgille/chromem-go/blob/v0.5.0/embed_ollama.go) with [example](https://github.com/philippgille/chromem-go/tree/v0.5.0/examples/rag-wikipedia-ollama)
 - [Kangaroo](https://github.com/dbkangaroo/kangaroo) (AI-powered SQL client and admin tool for popular databases)
Repository: vuejs/vue
Pull request: fix #5121: parse content in textarea as plaintext

Fix #5121 


Changed files:
filename: src/compiler/parser/html-parser.js
patch: @@ -46,7 +46,7 @@ let IS_REGEX_CAPTURING_BROKEN = false
 })
 
 // Special Elements (can contain anything)
-const isScriptOrStyle = makeMap('script,style', true)
+const isPlainTextElement = makeMap('script,style,textarea', true)
 const reCache = {}
 
 const decodingMap = {
@@ -72,8 +72,8 @@ export function parseHTML (html, options) {
   let last, lastTag
   while (html) {
     last = html
-    // Make sure we're not in a script or style element
-    if (!lastTag || !isScriptOrStyle(lastTag)) {
+    // Make sure we're not in a plaintext content element like script/style
+    if (!lastTag || !isPlainTextElement(lastTag)) {
       let textEnd = html.indexOf('<')
       if (textEnd === 0) {
         // Comment:
@@ -153,7 +153,7 @@ export function parseHTML (html, options) {
       var endTagLength = 0
       var rest = html.replace(reStackedTag, function (all, text, endTag) {
         endTagLength = endTag.length
-        if (stackedTag !== 'script' && stackedTag !== 'style' && stackedTag !== 'noscript') {
+        if (!isPlainTextElement(stackedTag) && stackedTag !== 'noscript') {
           text = text
             .replace(/<!--([\s\S]*?)-->/g, '$1')
             .replace(/<!\[CDATA\[([\s\S]*?)]]>/g, '$1')

filename: test/unit/modules/compiler/parser.spec.js
patch: @@ -506,4 +506,29 @@ describe('parser', () => {
     expect(ast.tag).toBe('div')
     expect(ast.children.length).toBe(0)
   })
+
+  it('parse content in textarea as text', () => {
+    const options = extend({}, baseOptions)
+
+    const whitespace = parse(`
+      <textarea>
+        <p>Test 1</p>
+        test2
+      </textarea>
+    `, options)
+    expect(whitespace.tag).toBe('textarea')
+    expect(whitespace.children.length).toBe(1)
+    expect(whitespace.children[0].type).toBe(3)
+    // textarea is whitespace sensitive
+    expect(whitespace.children[0].text).toBe(`
+        <p>Test 1</p>
+        test2
+      `)
+
+    const comment = parse('<textarea><!--comment--></textarea>', options)
+    expect(comment.tag).toBe('textarea')
+    expect(comment.children.length).toBe(1)
+    expect(comment.children[0].type).toBe(3)
+    expect(comment.children[0].text).toBe('<!--comment-->')
+  })
 })
Repository: ggml-org/llama.cpp
Pull request: mtmd : fix padding of n_tokens

fix #19929

`GGML_PAD` works only for powers of 2.

Changed files:
filename: tools/mtmd/mtmd-helper.cpp
patch: @@ -248,7 +248,7 @@ int32_t mtmd_helper_decode_image_chunk(
 
     int32_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk);
     int32_t i_batch = 0;
-    int32_t n_img_batches = GGML_PAD(n_tokens, n_batch) / n_batch;
+    int32_t n_img_batches = (n_tokens + n_batch - 1) / n_batch;
     decode_embd_batch batch_embd(encoded_embd, n_tokens, n_pos_per_embd, n_mmproj_embd);
 
     if (mtmd_decode_use_mrope(ctx)) {
Repository: ollama/ollama
Pull request: ci: be more aggressive on parallelism in build

This might help speed up release builds.

The old algorithm was trying not to overwhelm systems with hyperthreading.

Changed files:
filename: Dockerfile
patch: @@ -69,7 +69,7 @@ ARG OLLAMA_FAST_BUILD
 ARG VERSION
 RUN --mount=type=cache,target=/root/.ccache \
     if grep "^flags" /proc/cpuinfo|grep avx>/dev/null; then \
-        make -j $(expr $(nproc) / 2 ) dist ; \
+        make -j $(nproc) dist ; \
     else \
         make -j 5 dist ; \
     fi
Repository: facebook/react
Pull request: [ci] Remove compiler weekly release


No longer needed.


Changed files:
filename: .github/workflows/compiler_prereleases_weekly.yml
patch: @@ -1,23 +0,0 @@
-name: (Compiler) Publish Prereleases Weekly
-
-on:
-  schedule:
-    # At 10 minutes past 9:00 on Mon
-    - cron: 10 9 * * 1
-
-permissions: {}
-
-env:
-  TZ: /usr/share/zoneinfo/America/Los_Angeles
-
-jobs:
-  publish_prerelease_beta:
-    name: Publish to beta channel
-    uses: facebook/react/.github/workflows/compiler_prereleases.yml@main
-    with:
-      commit_sha: ${{ github.sha }}
-      release_channel: beta
-      dist_tag: beta
-      version_name: '19.0.0'
-    secrets:
-      NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
Repository: ollama/ollama
Pull request: runner: switch logging back to stderr

This puts the low-level runner logging back on stderr for consistency with prior releases

Changed files:
filename: llama/llama.go
patch: @@ -89,6 +89,7 @@ import (
 	_ "embed"
 	"errors"
 	"fmt"
+	"os"
 	"runtime"
 	"runtime/cgo"
 	"slices"
@@ -131,7 +132,7 @@ func llamaLog(level int32, text *C.char, _ unsafe.Pointer) {
 		return
 	}
 
-	fmt.Print(C.GoString(text))
+	fmt.Fprint(os.Stderr, C.GoString(text))
 }
 
 func GetModelArch(modelPath string) (string, error) {
Repository: ggml-org/llama.cpp
Pull request: server : fix ctx checkpoint restore logic

cont #19849
fix #19901

Prevent `n_past` from exceeding the available tokens in the checkpoint.

Changed files:
filename: tools/server/server-context.cpp
patch: @@ -2363,7 +2363,7 @@ struct server_context_impl {
                                             //printf("[DEBUG] `do_reset` was set to `true` after failing to restore a checkpoint");
                                         } else {
                                             pos_next = std::min(pos_next, std::max(it->pos_min + 1, it->pos_max));
-                                            n_past = slot.prompt.tokens.size_up_to_pos(pos_next);
+                                            n_past = std::min(slot.prompt.tokens.size_up_to_pos(pos_next), (size_t) it->n_tokens);
                                             SLT_WRN(slot, "restored context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", it->pos_min, it->pos_max, it->n_tokens, (float) checkpoint_size / 1024 / 1024);
                                         }
                                     }
Repository: electron/electron
Pull request: ci: ignore Dependabot workflow failures in audit

#### Description of Change

<!--
Thank you for your Pull Request. Please provide a description above and review
the requirements below.

Contributors guide: https://github.com/electron/electron/blob/main/CONTRIBUTING.md
-->

Dependabot has started having a consistent failure related to one package, and keeps tripping this audit. Ignore it since it's not the signal we're looking for with this audit.

#### Checklist
<!-- Remove items that do not apply. For completed items, change [ ] to [x]. -->

- [x] PR description included and stakeholders cc'd

#### Release Notes

Notes: none <!-- Please add a one-line description for app developers to read in the release notes, or 'none' if no notes relevant to app developers. Examples and help on special cases: https://github.com/electron/clerk/blob/main/README.md#examples -->


Changed files:
filename: .github/workflows/audit-branch-ci.yml
patch: @@ -74,6 +74,7 @@ jobs:
                       !message.startsWith("Process completed with exit code") &&
                       !message.startsWith("Response status code does not indicate success") &&
                       !message.startsWith("The hosted runner lost communication with the server") &&
+                      !message.startsWith("Dependabot encountered an error performing the update") &&
                       !/Unable to make request/.test(message) &&
                       !/The requested URL returned error/.test(message),
                   )
Repository: ollama/ollama
Pull request: llama: parse JSON schema using nlohmann::ordered_json

PR #8002 has handled the JSON within Go to ensure we could keep the schema as-is, without affecting the order of the properties. However, when parsed within the cpp wrapper, `nlohmann::json` was used instead of relying on `nlohmann::ordered_json`. This PR simply changes the parser for the ordered one in order to maintain the order.

Changed files:
filename: llama/sampling_ext.cpp
patch: @@ -49,7 +49,7 @@ int schema_to_grammar(const char *json_schema, char *grammar, size_t max_len)
 {
     try
     {
-        nlohmann::json schema = nlohmann::json::parse(json_schema);
+        nlohmann::ordered_json schema = nlohmann::ordered_json::parse(json_schema);
         std::string grammar_str = json_schema_to_grammar(schema);
         size_t len = grammar_str.length();
         if (len >= max_len)
Repository: huggingface/transformers
Pull request: Overwrite `get_decoder()` in AudioLLMs

# What does this PR do?

As per title, audio models need to call LM's `get_decoder()` because the model has no `base model`

Changed files:
filename: src/transformers/models/granite_speech/modeling_granite_speech.py
patch: @@ -322,6 +322,12 @@ def __init__(self, config: GraniteSpeechConfig):
 
         self.post_init()
 
+    def set_decoder(self, decoder):
+        self.language_model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.language_model.get_decoder()
+
     def set_input_embeddings(self, value):
         self.language_model.set_input_embeddings(value)
Repository: vuejs/vue
Pull request: fix #5591: keep ssr template interpolation whitespace-insensitive

Fix #5591 

**What kind of change does this PR introduce?** (check at least one)

- [x] Bugfix

**Does this PR introduce a breaking change?** (check one)

- [x] No

**The PR fulfills these requirements:**

- [x] It's submitted to the `dev` branch for v2.x (or to a previous version branch), _not_ the `master` branch
- [x] When resolving a specific issue, it's referenced in the PR's title (e.g. `fix #xxx[,#xxx]`, where "xxx" is the issue number)
- [x] All tests are passing: https://github.com/vuejs/vue/blob/dev/.github/CONTRIBUTING.md#development-setup
- [x] New/updated tests are included

Changed files:
filename: src/server/template-renderer/parse-template.js
patch: @@ -2,7 +2,7 @@
 
 const compile = require('lodash.template')
 const compileOptions = {
-  escape: /{{[^{]([\s\S]+?)[^}]}}/g,
+  escape: /{{([^{][\s\S]+?[^}])}}/g,
   interpolate: /{{{([\s\S]+?)}}}/g
 }
 

filename: test/ssr/ssr-template.spec.js
patch: @@ -330,5 +330,38 @@ describe('SSR: template option', () => {
         })
       })
     })
+
+    it('whitespace insensitive interpolation', done => {
+      const interpolateTemplate = `<html><head><title>{{title}}</title></head><body><!--vue-ssr-outlet-->{{{snippet}}}</body></html>`
+      const renderer = createRenderer({
+        template: interpolateTemplate
+      })
+
+      const context = {
+        title: '<script>hacks</script>',
+        snippet: '<div>foo</div>',
+        head: '<meta name="viewport" content="width=device-width">',
+        styles: '<style>h1 { color: red }</style>',
+        state: { a: 1 }
+      }
+
+      renderer.renderToString(new Vue({
+        template: '<div>hi</div>'
+      }), context, (err, res) => {
+        expect(err).toBeNull()
+        expect(res).toContain(
+          `<html><head>` +
+          // double mustache should be escaped
+          `<title>&lt;script&gt;hacks&lt;/script&gt;</title>` +
+          `${context.head}${context.styles}</head><body>` +
+          `<div data-server-rendered="true">hi</div>` +
+          `<script>window.__INITIAL_STATE__={"a":1}</script>` +
+          // triple should be raw
+          `<div>foo</div>` +
+          `</body></html>`
+        )
+        done()
+      })
+    })
   }
 })
Repository: facebook/react
Pull request: [compiler] Update publish tags


Adds missing tag.


Changed files:
filename: compiler/scripts/release/publish.js
patch: @@ -62,7 +62,7 @@ async function main() {
     .option('tag', {
       description: 'Tag to publish to npm',
       type: 'choices',
-      choices: ['experimental', 'beta'],
+      choices: ['experimental', 'beta', 'rc'],
       default: 'experimental',
     })
     .option('version-name', {
Repository: ollama/ollama
Pull request: llama: enable JSON schema key ordering for generating grammars

Will do a follow up PR for updates to the command line with format

Changed files:
filename: llama/grammar_test.go
patch: @@ -30,8 +30,6 @@ const issue7978JSONSchema = `{
 }`
 
 func TestIssue7978(t *testing.T) {
-	t.Skip("schema_to_grammar is broken; skipping until fixed")
-
 	g := SchemaToGrammar([]byte(issue7978JSONSchema))
 	if g == nil {
 		t.Fatal("failed to convert JSON schema to grammar")
@@ -54,16 +52,14 @@ func TestIssue7978(t *testing.T) {
 }
 
 func TestSchemaToGrammer(t *testing.T) {
-	t.Skip("schema_to_grammar is broken; skipping until fixed")
-
 	cases := []struct {
 		schema string
 		prefix []byte // nil is check as nil
 	}{
 		{`invalid`, nil},
 
 		// Simple heuristic/smoke test
-		{`{"type":"object"}`, []byte("object ::=")},
+		{`{"type":"object"}`, []byte("root ::= object")},
 	}
 
 	for _, c := range cases {

filename: llama/json-schema-to-grammar.cpp
patch: @@ -417,7 +417,7 @@ class SchemaConverter {
 private:
     std::function<json(const std::string &)> _fetch_json;
     bool _dotall;
-    std::map<std::string, std::string> _rules;
+    std::unordered_map<std::string, std::string> _rules;
     std::unordered_map<std::string, json> _refs;
     std::unordered_set<std::string> _refs_being_resolved;
     std::vector<std::string> _errors;

filename: llama/patches/0012-Maintain-ordering-for-rules-for-grammar.patch
patch: @@ -0,0 +1,22 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: ParthSareen <parth.sareen@ollama.com>
+Date: Wed, 11 Dec 2024 15:37:32 -0800
+Subject: [PATCH] Maintain ordering for rules for grammar
+
+---
+ common/json-schema-to-grammar.cpp | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp
+index dadc18c8..2a8dbd22 100644
+--- a/common/json-schema-to-grammar.cpp
++++ b/common/json-schema-to-grammar.cpp
+@@ -391,7 +391,7 @@ class SchemaConverter {
+ private:
+     std::function<json(const std::string &)> _fetch_json;
+     bool _dotall;
+-    std::map<std::string, std::string> _rules;
++    std::unordered_map<std::string, std::string> _rules;
+     std::unordered_map<std::string, json> _refs;
+     std::unordered_set<std::string> _refs_being_resolved;
+     std::vector<std::string> _errors;
Repository: vuejs/vue
Pull request: fix(types): improve typing for better completion

**What kind of change does this PR introduce?** (check at least one)

- [x] Other, please describe:
 Tweak definition file. Prefer user custom types over Vue builtin types for better completion item order. 

**Does this PR introduce a breaking change?** (check one)

- [ ] Yes
- [x] No

If yes, please describe the impact and migration path for existing applications:

**The PR fulfills these requirements:**

- [x] It's submitted to the `dev` branch for v2.x (or to a previous version branch), _not_ the `master` branch
- [x] When resolving a specific issue, it's referenced in the PR's title (e.g. `fix #xxx[,#xxx]`, where "xxx" is the issue number)
- [x] All tests are passing: https://github.com/vuejs/vue/blob/dev/.github/CONTRIBUTING.md#development-setup
- [ ] New/updated tests are included

**Other information:**


Changed files:
filename: types/vue.d.ts
patch: @@ -61,7 +61,7 @@ export interface Vue {
   $createElement: CreateElement;
 }
 
-export type CombinedVueInstance<Instance extends Vue, Data, Methods, Computed, Props> = Instance & Data & Methods & Computed & Props;
+export type CombinedVueInstance<Instance extends Vue, Data, Methods, Computed, Props> =  Data & Methods & Computed & Props & Instance;
 export type ExtendedVue<Instance extends Vue, Data, Methods, Computed, Props> = VueConstructor<CombinedVueInstance<Instance, Data, Methods, Computed, Props> & Vue>;
 
 export interface VueConstructor<V extends Vue = Vue> {
Repository: facebook/react
Pull request: [mcp] Also emit bailout messages with no loc


Not every bailout will contain a loc (could be synthetic)


Changed files:
filename: compiler/packages/react-mcp-server/src/index.ts
patch: @@ -158,7 +158,7 @@ server.tool(
         }
       }
     };
-    const errors: Array<{message: string; loc: SourceLocation}> = [];
+    const errors: Array<{message: string; loc: SourceLocation | null}> = [];
     const compilerOptions: Partial<PluginOptions> = {
       panicThreshold: 'none',
       logger: {
@@ -170,12 +170,10 @@ server.tool(
               detail.loc == null || typeof detail.loc == 'symbol'
                 ? event.fnLoc
                 : detail.loc;
-            if (loc != null) {
-              errors.push({
-                message: detail.reason,
-                loc,
-              });
-            }
+            errors.push({
+              message: detail.reason,
+              loc,
+            });
           }
         },
       },
@@ -279,17 +277,16 @@ server.tool(
         }
       }
       if (errors.length > 0) {
-        const errMessages = errors.map(err => {
-          if (typeof err.loc !== 'symbol') {
+        return {
+          content: errors.map(err => {
             return {
               type: 'text' as const,
-              text: `React Compiler bailed out:\n\n${err.message}@${err.loc.start.line}:${err.loc.end.line}`,
+              text:
+                err.loc === null || typeof err.loc === 'symbol'
+                  ? `React Compiler bailed out:\n\n${err.message}`
+                  : `React Compiler bailed out:\n\n${err.message}@${err.loc.start.line}:${err.loc.end.line}`,
             };
-          }
-          return null;
-        });
-        return {
-          content: errMessages.filter(msg => msg !== null),
+          }),
         };
       }
       return {
Repository: ggml-org/llama.cpp
Pull request: kv-cache : fix can_shift() check to take into account M-RoPE

fix #19915

KV cache shift is not supported with M-RoPE (yet).

Changed files:
filename: src/llama-kv-cache.cpp
patch: @@ -978,6 +978,9 @@ bool llama_kv_cache::get_can_shift() const {
     if (model.arch == LLM_ARCH_STEP35) {
         return false;
     }
+    if (hparams.n_pos_per_embd() > 1) {
+        return false;
+    }
     return true;
 }