DOĞAL DİL İŞLEME ROADMAP

# spaCy — modern NLP kütüphanesi import spacy nlp = spacy.load("tr_core_news_md") # Türkçe model doc = nlp("Ahmet dün İstanbul'dan geldi ve bana güzel haberler verdi.") for token in doc: print(f"{token.text:15} {token.lemma_:15} {token.pos_:8} {token.dep_}") # Named Entity Recognition for ent in doc.ents: print(f"{ent.text:20} → {ent.label_}") # Ahmet → PERSON, İstanbul → GPE # TF-IDF — en iyi klasik vektörleme from sklearn.feature_extraction.text import TfidfVectorizer vect = TfidfVectorizer(max_features=10000, ngram_range=(1,2), sublinear_tf=True) X = vect.fit_transform(corpus) # sparse matrix top_feats = vect.get_feature_names_out()[:10]

# Word2Vec mantığı — sıfırdan anlama # "Kral - Erkek + Kadın ≈ Kraliçe" from gensim.models import Word2Vec model = Word2Vec( sentences=tokenized_corpus, vector_size=100, # embedding boyutu window=5, # bağlam penceresi min_count=2, # min frekans workers=4, # paralel eğitim sg=1 # Skip-gram (vs CBOW) ) model.wv.most_similar("türkiye", topn=5) model.wv.most_similar(positive=["kral","kadın"], negative=["erkek"]) # LSTM ile sentiment analysis import torch import torch.nn as nn class SentimentLSTM(nn.Module): def __init__(self, vocab_sz, emb_dim, hidden, n_layers): super().__init__() self.embed = nn.Embedding(vocab_sz, emb_dim, padding_idx=0) self.lstm = nn.LSTM(emb_dim, hidden, n_layers, batch_first=True, dropout=0.3, bidirectional=True) self.fc = nn.Linear(hidden * 2, 1) def forward(self, x): e = self.embed(x) out, (h, _) = self.lstm(e) h_cat = torch.cat([h[-2], h[-1]], dim=1) return torch.sigmoid(self.fc(h_cat).squeeze())

# Scaled Dot-Product Attention — Transformer'ın kalbi import torch, math import torch.nn.functional as F def attention(Q, K, V, mask=None, dropout=None): # Q, K, V: (batch, heads, seq_len, d_head) d_k = Q.size(-1) scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(d_k) if mask is not None: scores = scores.masked_fill(mask == 0, float("-inf")) attn = F.softmax(scores, dim=-1) # dikkat ağırlıkları if dropout: attn = dropout(attn) return torch.matmul(attn, V), attn # output + weights # "attention head her şeye eşit dikkat etmez;" # "konuyla ilgili kelimelere daha fazla dikkat eder"

# HuggingFace — NLP mühendisinin en önemli silahı from transformers import (AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer, DataCollatorWithPadding) from datasets import load_dataset from peft import LoraConfig, get_peft_model, TaskType import evaluate # Türkçe BERT fine-tuning — sentiment analysis model_name = "dbmdz/bert-base-turkish-cased" tokenizer = AutoTokenizer.from_pretrained(model_name) model = AutoModelForSequenceClassification.from_pretrained( model_name, num_labels=3 # pozitif/negatif/nötr ) # LoRA — büyük modeli az parametreyle fine-tune lora_cfg = LoraConfig( task_type=TaskType.SEQ_CLS, r=16, lora_alpha=32, lora_dropout=0.1, target_modules=["query", "value"] # attention matrislerine ) model = get_peft_model(model, lora_cfg) model.print_trainable_parameters() # ~0.5% parametre! training_args = TrainingArguments( output_dir="./turkce-sentiment", num_train_epochs=3, learning_rate=2e-4, per_device_train_batch_size=16, weight_decay=0.01, eval_strategy="epoch", fp16=True, load_best_model_at_end=True, report_to="wandb" ) trainer = Trainer(model=model, args=training_args, train_dataset=train_ds, eval_dataset=val_ds, tokenizer=tokenizer) trainer.train()

# RAG — Retrieval Augmented Generation # "Modeli yeniden eğitmek yerine bilgiyi dışarıdan ver" from langchain.document_loaders import PyPDFLoader from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_openai import OpenAIEmbeddings, ChatOpenAI from langchain_chroma import Chroma from langchain.chains import RetrievalQA # 1. PDF yükle ve parçalara böl docs = PyPDFLoader("belge.pdf").load() chunks = RecursiveCharacterTextSplitter(chunk_size=512, chunk_overlap=64).split_documents(docs) # 2. Embedding → vektör veritabanı db = Chroma.from_documents(chunks, OpenAIEmbeddings()) # 3. RAG zinciri rag = RetrievalQA.from_chain_type( llm=ChatOpenAI(model="gpt-4o-mini"), retriever=db.as_retriever(search_kwargs={"k": 4}), return_source_documents=True ) yanit = rag.invoke("Bu belgede ne anlatılıyor?")

# FastAPI + streaming response — modern NLP API from fastapi import FastAPI from fastapi.responses import StreamingResponse from transformers import pipeline, TextIteratorStreamer import threading, torch app = FastAPI(title="NLP API") model = pipeline("text-generation", model="microsoft/phi-2", device=0 if torch.cuda.is_available() else -1) @app.post("/generate") async def generate(prompt: str, max_new_tokens: int = 256): streamer = TextIteratorStreamer(model.tokenizer, skip_special_tokens=True) thread = threading.Thread( target=model, args=([prompt],), kwargs={"max_new_tokens": max_new_tokens, "streamer": streamer} ) thread.start() return StreamingResponse(streamer, media_type="text/plain") @app.post("/sentiment") async def sentiment(text: str): clf = pipeline("sentiment-analysis", model="dbmdz/bert-base-turkish-cased") return clf(text)[0]