adds notebook to scrap jobs from job street

.gitignore

@@ -20,6 +20,7 @@ logs/**
 # also keep the example dataset
 !data/01_raw/*.csv
 !data/02_processed/**
+!data/02_processed/**/**
 
 # also keep all .gitkeep files
 !.gitkeep

conf/params.yml

@@ -1,22 +1,32 @@
 # Data
 ingestion_data_dir: ["data", "02_processed"]
 ingestion_metadata_dir: ["data", "02_processed", "metadata"]
+job_titles: ["Data Engineer", "Data Scientist", "Data Analyst", "Machine Learning Engineer"]
 
 # Embeddings
 embedding_model_name: "sentence-transformers/all-mpnet-base-v2"
 embedding_dir: ["data", "03_indexed"]
 
 # app
-APP_TITLE: "ResumeWorth"
-APP_DESCRIPTION: |
-    Discover Your Market Value!
+app_config:
+    host: "0.0.0.0"
+    port: 7860
+app_backend:
+    min_resume_size: 1000
+app_frontend:
+    title: "ResumeWorth"
+    description: |
+        Discover Your Market Value!
 
-    Unlock the true value of your experience with ResumeWorth. 
+        Unlock the true value of your experience with ResumeWorth. 
 
-    Simply enter your job title, upload your resume, and let our advanced AI analyze your professional background. 
-    
-    In moments, you'll receive an estimated salary, tailored to your unique skills and experience. 
-# Empower your job search with insights that put you a step ahead.
-JOB_TITLES: ["Data Engineer", "Data Scientist", "Data Analyst", "Machine Learning Engineer"]
-HOST: "0.0.0.0"
-PORT: 7860
+        Simply enter your job title, upload your resume, and let our advanced AI analyze your professional background. 
+        
+        In moments, you'll receive an estimated salary, tailored to your unique skills and experience.
+    # Empower your job search with insights that put you a step ahead.
+    jobs: ["Data Engineer", "Data Scientist", "Data Analyst", "Machine Learning Engineer"]
+    messages:
+        job_title_not_found: "You must choose one of the roles in the Job Title section!"
+        salary_not_found: |
+            There is no good match between resume and available job vacancies.
+            We recommend that you submit an extended version of your resume.

data/01_raw/data_jobs_scraped_from_jobstreet_2024-04-07.csv

@@ -0,0 +1,639 @@
+,job_id,job_title,company,location,category,sub_category,job_type,salary,description
+0,74727704,Machine Learning Engineer (Artificial Intelligence),Private Advertiser,Kuala Lumpur,Science & Technology,"Mathematics, Statistics & Information Sciences",Full time,"RM 4,000 – RM 6,000 per month","Design, develop, and deploy machine learning models and algorithms for complex and unique datasets, using various techniques such as mathematical modeling, scikit-learn, NLP, CNN, RNN, DL, RL, Transformers, GAN, LLM, RAG
+Collaborate with cross-functional teams to extract insights, identify business opportunities and provide data-driven recommendations
+Stay up-to-date with the latest machine learning and AI techniques and tools
+Communicate complex technical concepts to non-technical stakeholders in an easy-to-understand manner
+Bachelor's degree or higher in Computer Science, Mathematics, Statistics, Actuarial Science, Informatics, Information Science or related fields
+Strong analytical skills and attention to detail
+Participation in Kaggle, Mathematics Olympiad or similar competitions is a plus
+Excellent programming skills in Python, R, Java, or C++
+Familiar with ML frameworks such as Tensorflow, Keras, PyTorch, MLFlow, AutoML, TensorRT, CUDA
+Excellent communication and collaboration skills
+Experience with designing, training, and deploying machine learning models
+Customer centric and committed to deliver the best AI results to customers
+What's your expected monthly basic salary?
+
+
+Role descriptions
+
+
+Salary insights
+
+
+Tools to help you prepare for jobs
+
+"
+1,74850440,Data Scientist,PHENOMENAL UAM Sdn Bhd,Kuala Lumpur,Science & Technology,"Mathematics, Statistics & Information Sciences",Full time,"RM 5,000 – RM 7,500 per month","Utilizing machine learning techniques to analyze and interpret complex data sets.
+Developing AI models and algorithms to solve specific business problems and improve operational efficiency.
+Collaborating with engineers and software developers to integrate AI solutions into our products and services.
+Conducting exploratory data analysis to identify trends, patterns, and insights.
+Designing and implementing experiments to validate model performance and accuracy.
+Communicating findings and insights to stakeholders through visualizations, presentations, and reports.
+Staying abreast of the latest advancements in AI and machine learning technologies, and evaluating their potential applications within our organization.
+Bachelor's degree in Computer Science, Data Science, Statistics, Mathematics, or a related field. Advanced degree (Master's or Ph.D.) preferred.
+Proven experience (X years) working as a Data Scientist or AI Engineer, with a strong focus on developing and implementing machine learning algorithms and AI models.
+Proficiency in programming languages such as Python, R, or Java, along with experience using libraries and frameworks such as TensorFlow, PyTorch, or scikit-learn.
+Solid understanding of statistical analysis, hypothesis testing, and experimental design.
+Experience with data visualization tools such as Matplotlib, Seaborn, or Tableau.
+Strong problem-solving skills and the ability to translate business requirements into analytical solutions.
+Excellent communication and collaboration skills, with the ability to effectively convey technical concepts to both technical and non-technical stakeholders.
+Familiarity with cloud computing platforms (e.g., AWS, Azure, Google Cloud) and big data technologies (e.g., Hadoop, Spark) is a plus.
+Passion for innovation and a keen interest in staying updated with the latest trends and advancements in AI and machine learning."
+2,74946043,Data Engineer Lead,KK Group Of Companies,Bandar Tasik Selatan,Information & Communication Technology,Engineering - Software,Full time,"RM 7,000 – RM 8,000 per month","Making large and/or complex data more accessible, understandable and usable
+Transforming, improving and integrating data, depending on the business requirements
+Delivering the data in a useful and appealing way to users
+Work with noisy, dirty, and unstructured data, data cleansing, scraping unstructured data, and converting into structured data
+Build and maintain data pipelines, ETL, databases and applications to acquire, store and manage data
+Choose, design, test and implement data software and solutions for projects and enterprise
+Selecting and integrating any Big Data tools and frameworks required to provide requested capabilities
+Monitoring performance and advising necessary infrastructure changes to optimize performance and cost
+Architect, develop and implement end-to-end automated data flows for data and analytical needs
+Perform data profiling, data quality assessment, data cleansing and data transformation
+Recommend appropriate solutions architecture that is highly scalable based on business requirement
+Create date tools for data scientists and data visualization team members
+Builds the infrastructure required for optimal extraction, transformation, and loading for data from a wide variety of data sources using SQL, noSQL and cloud (AWS / Azure / Google / Alibaba / DigitalOcean) technologies
+Benchmark, evaluate and improve the scalability, robustness, efficiency and performance of big data platforms and applications
+Engage in the delivery of analytics assignments for business users involving, amongst others; data preparation, data visualization, creating business insights etc
+Contribute and facilitate workshops/interviews bridging the gap between data analytics and business outcomes
+Provide strategic and tactical advice to business units by translating analytical solutions into out-of-the-box execution recommendations addressing business objectives and impacting bottom line results
+Combine your technical expertise with the ability to architect, deploy and oversee integrated analytics/business intelligence/technology solutions
+Build case for analytics through need analysis and business case (quantify likely benefits)
+Consult with leaders of business teams and manage expectations, meet project and deliverable timelines, ensure accurate and timely completion of deliverables
+Collaborate with several other teams and roles across the organization on requirements gathering and solution deployment
+Make analytical product, service or process decisions likely to impact multiple groups of employees and/or customers (internal or external)
+Manage technology solution portfolios, and identifying new technology approaches to solve business problems with a strong focus on leveraging enterprise data
+Contribute to the building and management of the analytics infrastructure
+Work with management team on staffing needs, strategic technology trend identification, monitoring project progress and completion, issue resolution and technical solutions related to data, data flow and advising on algorithm deployment
+Collaborate with analytics practitioners, business analysts and methodologists to design and oversee the development of working prototypes of technical solutions
+Technical support and lead for the Data Engineer team
+Work with project teams to assist with data-related technical issues and support their data infrastructure needs
+What's your expected monthly basic salary?
+How many years' experience do you have as a Data Engineer?
+How many years' experience do you have as a Tableau Specialist?
+How many years' experience do you have as a Power BI Developer?
+Have you completed an Amazon Web Services Developer certification?
+Have you completed a Cloudera Certified Professional Data Engineer certification?
+How many years' experience do you have as an ETL Data Warehouse Programmer?
+How much notice are you required to give your current employer?"
+3,74915330,Data Engineer (Python),Inter Island Manpower Pte Ltd,Kuala Lumpur,Information & Communication Technology,Engineering - Software,Full time,"RM 6,000 – RM 7,000 per month","Bachelor's or Master's degree in Computer Science, Engineering, or related fields.
+1-3 years as a Data Engineer with expertise in data engineering concepts.
+Hands-on experience with Azure or AWS cloud services, Python/Java/Scala, and data processing frameworks.
+Strong knowledge of data modeling, database technologies, and problem-solving skills.
+Excellent communication, collaboration, and adaptability to fast-paced environments.
+Design, develop, and maintain data pipelines for diverse sources into data lake/warehouse.
+Collaborate with data scientists for analysis and reporting needs.
+Research and evaluate data sources for property development.
+Develop web-scraping scripts and automate pipeline deployment.
+Utilize Azure, AWS, and relevant tech for data solutions.
+Integrate data between systems/platforms, monitor, and optimize pipelines.
+Ensure data security, compliance, and quality while staying updated on data tech trends.
+How many years' experience do you have as a Data Engineer?
+Which of the following programming languages are you experienced in?"
+4,74891040,Principal Data Engineer,Agensi Pekerjaan Hays (Malaysia) Sdn Bhd,Petaling,Information & Communication Technology,Database Development & Administration,Full time,$10000.0 - $14000.0 p.m.,"Take ownership of the end-to-end data solution development and operation, including modelling, ETL, scripting, visualisation, testing, deployment, etc.
+Drive maturity of data infrastructure and architecture by designing, implementing, and maintaining the ingestion, ETL, storage, consumption, archival and disposal process and structure for product and organisation.
+Build data products, including APIs, visualisation tools, analytics and machine learning platforms.
+Utilise ETL to provide actionable insights and intelligence, serving both product offerings and internal data consumption for business functions.
+Coach and mentor the junior team members to develop and improve both technical and non-technical capabilities.
+At least 3 years of experience in a Data Engineering role.
+Strong command of SQL, Python, Data Modelling, Data Warehousing, ETL/ELT.
+Strong foundation in cloud data services such as AWS Redshift, Azure Synapse, BigQuery, Snowflakes, etc.
+A competitive salary based on performance and output.
+Excellent career support and leadership training.
+Exposure to the growing fintech industry and room to grow at an elevated rate."
+5,74923865,Data Analyst,MONKEYCLOUD MEDIA SDN. BHD.,Petaling,Information & Communication Technology,Help Desk & IT Support,Full time,"RM 3,000 – RM 4,500 per month","Collect and compile data from different sources, including databases, and spreadsheets.
+Filtering and cleaning data to ensure accuracy.
+Analyze data to identify trends, patterns, and anomalies, and provide actionable insights to support in various team decision-making.
+Develop and maintain dashboards and reports to track key project performance metrics, including progress and target achievement.
+Conduct ad-hoc data analysis to address specific project management questions or issues.
+Assist in the preparation of presentations and reports for project status updates and executive-level reporting.
+Diploma or Degree in any related field
+Experience in data analysis, business analysis or similar role, with a strong focus on data manipulation and analysis is preferred.
+Familiarity with data analysis tools and concepts, with experience in Power BI being a strong advantage.
+2-3 years working experience as a Business Intelligence Analyst/Data Analyst or any position that related to data and business analytics with strong statistical background.
+Preferably with prior experience in the FMCG industry.
+Experience in data analysis, business analysis or similar role, with a strong focus on data manipulation and analysis is preferred.
+Proficiency in data analysis tools and software, such as Power BI, Microsoft Excel (Pivot, Macro) or similar tools.
+Strong organizational skills and the ability to manage multiple tasks and priorities simultaneously.
+A collaborative and team-oriented mindset with the ability to work effectively in a cross-functional environment.
+Excellent communication and presentation skills, with the ability to convey complex information clearly and effectively.
+Strong analytical and problem-solving skills, with the ability to translate data into actionable insights.
+Monday to Friday
+ : 8.30a.m to 5.30p.m 
+How many years' experience do you have as a data analyst?
+How many years' experience do you have as a Power BI Consultant?
+How would you rate your Mandarin language skills?
+Which of the following data analytics tools are you experienced with?
+Which of the following Relational Database Management Systems (RDBMS) are you experienced with?"
+6,74879920,Data Analyst (Intern),MONT ROYALE COLLEGE,Sri Hartamas,Science & Technology,"Mathematics, Statistics & Information Sciences",Contract/Temp,RM 600 per month,"Collect, organize, and analyze data from multiple sources to identify trends, patterns, and insights.
+Develop and maintain databases and data systems necessary for data analysis and reporting.
+Generate regular reports and dashboards to communicate key findings and metrics to stakeholders.
+Collaborate with departments to understand their data needs and provide analytical support to inform strategic initiatives.
+Conduct thorough data quality assessments and implement solutions to address any discrepancies or issues.
+Stay updated on industry trends and best practices in data analysis and visualization techniques.
+Bachelor's degree in Computer Science, Statistics, Mathematics, Economics, or related field. (Master's degree preferred)
+Proven experience working as a data analyst or in a similar role.
+Proficiency in data analysis tools and software such as Excel, SQL, Python, R, or Tableau.
+Strong analytical and problem-solving skills with the ability to translate complex data into actionable insights.
+Excellent communication and presentation skills, with the ability to effectively convey technical information to non-technical stakeholders.
+Detail-oriented with a commitment to accuracy and data integrity.
+Ability to work independently and collaboratively in a fast-paced environment.
+What's your expected monthly basic salary?"
+7,74866336,Manager - Data Engineer,Elken Sdn Bhd,Petaling,Science & Technology,"Mathematics, Statistics & Information Sciences",Full time,"RM 8,000 – RM 12,000 per month","Provide database operation support including monitoring, performance tuning, backup and restoration, disaster recovery, troubleshooting, replication setup and capacity management.
+Responsible for installing, configuration, upgrading and maintaining multiple environments including on premise and cloud databases.
+Provide recommendation for future direction of database technology and infrastructure setup.
+Deploy database objects and data patches when required by development team.
+Assist developers with complex query tuning and best practice suggestion.
+Database environment setup for Development, UAT and Production.
+Database migration for various platform.
+Build automation script for database tasks.
+Collaborate with Data Analysts, and other stakeholders to understand data requirements and provide data solutions to meet their needs.
+Bachelor’s Degree in Computer Science or equivalent in related field.
+At least 3 years working experience in related field.
+Possess professional database certifications such as Microsoft Certified, Oracle Certified will be added advantage.
+Experience with RDBMS Administration such as MS SQL, My SQL MariaDB, PostgresSQL and Oracle.
+Experience with AWS RDS Cloud database environment administration.
+Experience with Windows Server and Linux environments.
+Experience with Powershell scripting, Linux command line.
+Able to converse in 
+Mandarin
+ as needed to support oversea team like Hong Kong, China and Taiwan.
+What's your expected monthly basic salary?
+Which of the following Relational Database Management Systems (RDBMS) are you experienced with?"
+8,74789886,Junior Data Analyst,Oriental Fortune Trading Malaysia Sdn Bhd,Petaling,Information & Communication Technology,Business/Systems Analysts,Full time,"RM 3,000 – RM 4,300 per month","Data Collection and Cleaning:
+ Collect and compile data from various sources, ensuring data accuracy and completeness. Perform data cleaning and transformation as necessary to prepare it for analysis.
+Data Analysis:
+ Assist in conducting data analysis using statistical methods, data visualization tools, and data analysis software. Summarize findings in clear and concise reports.
+Data Visualization:
+ Create visually appealing and informative charts, graphs, and dashboards to present data insights to stakeholders. Use tools such as Tableau, Power BI, or Excel for visualization.
+Report Generation:
+ Prepare regular and ad-hoc reports to communicate key metrics and findings to the team and management. Ensure reports are accurate and delivered on time.
+Data Quality Assurance:
+ Help maintain data quality standards by identifying and resolving data discrepancies or inconsistencies. Collaborate with other team members to establish data quality protocols.
+Collaboration: 
+Work closely with cross-functional teams to understand data requirements and deliver actionable insights.
+Research and Learning:
+ Stay up-to-date with industry trends and best practices in data analysis. Continuously expand your knowledge and skills in data analytics techniques and tools.
+Bachelor's degree in a relevant field such as Finance, Data Science, Statistics, Mathematics, Computer Science, or a related discipline.
+Candidate with 1-2 years quantitative analysis or finance background are preferred; 
+fresh graduates are welcome as well.
+Required language(s): 
+Fluent in English & Mandarin
+ are preferred because the role requires candidates to interact with Mandarin-speaking clients.
+Strong analytical skills and a passion for working with data.
+Proficiency in data analysis tools and programming languages such as Excel, SQL, Python, or R.
+Familiarity with data visualization tools (e.g., Tableau, Power BI) is a plus.
+Excellent communication skills, with the ability to translate complex data into understandable insights.
+Detail-oriented and capable of maintaining data accuracy and integrity.
+Highly organized and able to manage multiple tasks and deadlines effectively.
+Team player with a collaborative mindset and the ability to work well in a diverse team environment.
+Eagerness to learn and adapt to new tools, techniques, and technologies in the field of data analysis.
+Possess own car.
+What's your expected monthly basic salary?
+How many years' experience do you have as a data analyst?
+Which of the following languages are you fluent in?"
+9,74703677,Junior Data Analyst,Private Advertiser,Bangsar South,Information & Communication Technology,Business/Systems Analysts,Full time,"RM 3,500 – RM 4,300 per month","Analyze and manage data related to E-sports events, focusing on Key Performance Indicators (KPIs).
+Create daily reports highlighting discrepancies or errors identified during the analysis process.
+Chart and conduct in-depth analyses of team performance in E-sports events.
+Real-time monitoring and analysis of data, including the production of charts and performance evaluations.
+Maintain daily event reports with important metrics for overall performance.
+Collaborate with team members, ensuring smooth operations and providing support.
+Fluency in Mandarin preferred for client interaction.
+Strong analytical and critical thinking skills.
+Ability to work independently under pressure and manage multiple projects with tight schedules.
+Flexibility for shifts and overnight work.
+Passion for E-sports is a plus.
+Fresh graduates are welcome to apply.
+13 months’ Salary
+Quarterly and annual bonuses
+Monthly meal, shift, and parking allowances
+Monthly team gatherings & birthday celebrations
+Unlimited snacks & beverages
+Team building activities & outings
+Medical insurance
+14 days of annual leave
+Birthday leave
+8 hours of work per day
+EPF, SOCSO, EIS contributions
+Which of the following languages are you fluent in?"
+10,74602585,Data Analyst/Scientist,PERSOLKELLY Workforce Solutions Malaysia Sdn Bhd,Kulim District,Science & Technology,"Mathematics, Statistics & Information Sciences",Contract/Temp,"RM 3,500 – RM 5,000 per month","What's your expected monthly basic salary?
+How many years' experience do you have as a data analyst?
+How many years' experience do you have using SQL queries?"
+11,74586835,Data Engineer,Michael Page International (Malaysia) Sdn Bhd,Kuala Lumpur,Information & Communication Technology,Database Development & Administration,Full time,$36k - $64k p.a. + Bonus + Other benefits,"Designing and implementing Data pipelines and ETL processes from various sources into our Data Warehouse
+Ensure that data is cleaned, standardized, and properly structured to minimize data inconsistencies and gaps
+Create and maintain data pipelines to facilitate the smooth flow of data throughout the organization
+Work closely with cross-functional teams to identify data integration needs and implement solutions that enable seamless data movement
+Establish data quality standards and processes to monitor and maintain the accuracy and integrity of data within Big Query and other data systems
+Implement data validation techniques to identify and rectify invalid or incomplete data entries
+Design and maintain data schemas to ensure consistency and efficiency in data storage and retrieval
+Collaborate with Tech and Business teams to refine data models and adapt to changing business needs
+Drive data-centric decision-making within the organization
+Collaborate with stakeholders to foster a data-driven culture by promoting data literacy and usage
+Bachelor's degree in Computer Science, Engineering or a related field
+Proven experience as a Data Engineer or similar role, with a track record of implementing successful data solutions
+Proficiency in ETL (Extract, Transform, Load) processes and data integration techniques
+Strong knowledge of database systems and data warehousing concepts
+Experience with Google Cloud Platform (GCP) services"
+12,74513015,Senior Data Scientist (Full Stack),Ambition Group Malaysia Sdn Bhd,Petaling,Consulting & Strategy,Other,Full time,Attractive Remuneration Package,"Utilize technical proficiency for data gathering, cleaning, and preprocessing from varied sources.
+Develop, train, and assess machine learning models for solving business problems like predictive analytics and natural language processing.
+Conduct statistical analysis, hypothesis testing, and data-driven experimentation; stay updated on emerging trends in data science and machine learning
+Degree/Master in Computer Science, Mathematics, Statistics.
+Experienced in all aspects of the data science lifecycle, including problem formulation, data collection and preprocessing, exploratory data analysis, feature engineering, model selection and training, model evaluation.
+Familiar working with cloud-based big data platforms (Azure, GCP, AWS)
+A minimum 3 years of hands on experience with Python.
+Excellent communication skills in Bahasa Melayu and English."
+13,74636692,Data Analyst,Agensi Pekerjaan Talent Recruit Sdn. Bhd.,Kuala Selangor District,Information & Communication Technology,Business/Systems Analysts,Full time,"RM 4,000 – RM 5,000 per month","Analyze and interpret key indicators in e-commerce
+Provide valuable insights and identify trends based on the sales data and suggest recommendations and key action plans 
+Provide competitor analysis 
+Evaluation of the effectiveness of sales and marketing tools and mechanics
+Creating and developing tools for analysis and information processing 
+Presenting the results of reports and graphical modeling of data 
+Work closely with the e-Commerce sales team to increase sales and revenue 
+Develop and maintain data pipelines to ensure data accuracy, integrity, and availability
+Develop and maintain e-Commerce dashboards as per client’s requests
+Providing insight to the e-Commerce sales team to optimize marketing expenses
+Data Review: reviewing and validating data from various sources (e.g. Lazada, Shopee, etc.) to ensure data quality and accuracy
+Diploma or Bachelor’s degree in Data Science, Statistics, Computer Science, or a related field 
+Min 
+1 - 2 years of experience as a Data Analyst/Business Analyst/Category Management preferably in E-Commerce or FMCG industry
+Advanced Excel and data mining skills 
+Proficient in data visualization tools (e.g. Power BI) and database query languages (e.g. SQL) 
+Knowledge of Cloud Computing is an added advantage (e.g. Azure) 
+Familiarity with Azure Data Factory and Azure Synapse Pipelines is an added advantage 
+Ability to work independently and take the lead in projects 
+Strong presentation skills
+Permanent employment
+Working hours : Mon - Fri (9am – 6pm) *
+Saturday half day only when required by Management during certain peak events (very occasional)
+Working location : Puncak Alam
+What's your expected monthly basic salary?"
+14,74601731,Data Analyst Internship,Air Energy Consulting (Malaysia) Sdn Bhd,Kuala Lumpur,Administration & Office Support,Records Management & Document Control,Contract/Temp,"RM 900 – RM 1,000 per month","Review and update on Bullhorn data for Asia placements.
+Work with the Service Team Australia on updating the Bullhorn data for data accuracy.
+Work closely with the Quality Coordinator on the accuracy and completeness of the data (Client’s PO/WO and Contractors’ employment packages) within the Airswift systems.
+Work with the relevant parties to ensure that any errors are corrected promptly.
+Administrator and investigate client, assignee and consultant surveys.
+Data analysis – accuracy report that was generated for the entire Asia Pacific Region.
+Consolidate data and dashboards for distribution to SVPs and Global Operations as required.
+Manage the Business Continuity plan and Office Safety for the super region and specific country.
+Work with other members of the Regional Commercial Team, ensuring that all requests for review, questions and queries are addressed, and escalating to the relevant people when required.
+The entire process flow of how a manpower/workforce solution provider organisation functions (Sales/BD -> Recruitment -> Onboarding -> Payroll -> Billing).
+Diploma or Degree in Business Administration or Management or equivalent.
+Knowledge of relevant regulatory requirements in the region.
+Excellent technical skills including all Microsoft Office products and understanding of statistics.
+Ability to problem solve with high attention to detail.
+Excellent written and verbal communication and interpersonal skills.
+Must have a sense of urgency in completing tasks.
+Client service orientated with good teamwork skills.
+Ownership and willingness to accept responsibility.
+Virtual Onboarding Events.
+Team-driven environment fostering a supportive culture with a strong emphasis on work-life balance.
+Cheers! where your peers can nominate you for outstanding performance.
+Coaching and Training are provided.
+Growth
+ - In life and business, one must grow to flourish and achieve high ambitions. Growth requires change, challenge, risk and sacrifice - we will always choose growth.
+Safety
+ - We create a safe and inclusive environment to protect the physical and psychological well-being of everyone we work with.
+Excellence
+ - We deliver, holding ourselves accountable for results. Our customers see excellence in everything we do.
+Integrity
+ - We are ethical, open, honest and authentic. People trust us to do the right thing for the right reason.
+Which of the following statements best describes your right to work in Malaysia?"
+15,74360323,Data Scientist,TALENTBANK SDN BHD,Kuala Lumpur,Science & Technology,"Mathematics, Statistics & Information Sciences",Full time,"RM 6,800 – RM 10,200 per month","Which of the following programming languages are you experienced in?
+What's your expected monthly basic salary?
+Which of the following data analytics tools are you experienced with?
+How many years' experience do you have as a data scientist?"
+16,74433737,Data Engineer,Michael Page International (Malaysia) Sdn Bhd,Selangor,Sales,Other,Full time,$96k - $97k p.a.,"Attractive remuneration package
+Be part of a globally recognised organisation
+Enhance your development skills"
+17,74323540,Data Scientist,AGENSI PEKERJAAN LEWIS TALENT CONSULTING SDN. BHD.,Petaling,Science & Technology,"Mathematics, Statistics & Information Sciences",Full time,"RM 6,000 – RM 8,500 per month","Understanding the data information, investigating and preprocessing vast datasets to unveil significant features and insights.
+Design and deploy predictive models employing advanced statistical and machine learning techniques to address complex business issues.
+Generate compelling visualizations to illustrate complex analyses and insights to both technical and non-technical audiences.
+Engage closely with multidisciplinary teams to grasp business objectives and partake in data-driven projects.
+Keep abreast of the latest data science advancements and enhance the organization's data science methodologies.
+At least possess a Master's or Ph.D. in Data Science, Statistics, Computer Science, or related fields.
+Demonstrated experience as a Data Scientist, supported by a portfolio of successful data science projects.
+Expertise in Python or R, including familiarity with relevant libraries and frameworks.
+Solid foundation and experience in applying machine learning algorithms to practical problems.
+Strong interpersonal and communication capabilities for effective teamwork with cross-functional groups.
+Commitment to staying informed about industry trends, data science techniques, and tools. Engage in ongoing training and development programs as necessary.
+Expertise in analyzing extensive, complex datasets to extract valuable insights.
+Proficiency in applying statistical and machine learning methods to solve real-world challenges.
+Competence in programming languages like Python or R for data analysis and modeling.
+Strong ability in creating visualizations with tools such as Tableau, Power BI, or similar platforms.
+Creative problem-solving aptitude for tackling complex business issues with data-driven solutions.
+Exceptional ability to articulate complex findings and insights clearly.
+How many years' experience do you have as a data scientist?
+Which of the following programming languages are you experienced in?
+Which of the following data analytics tools are you experienced with?
+How would you rate your Mandarin language skills?
+Which of the following languages are you fluent in?"
+18,73981733,Big Data Engineer - Cloud,Private Advertiser,Kuala Lumpur,Information & Communication Technology,Engineering - Software,Contract/Temp,"RM 9,000 – RM 13,000 per month","Which of the following statements best describes your right to work in Malaysia?
+What's your expected monthly basic salary?
+
+
+Role descriptions
+
+
+Salary insights
+
+
+Tools to help you prepare for jobs
+
+"
+19,74324430,Data Scientist Assistant Manager,AGENSI PEKERJAAN LEWIS TALENT CONSULTING SDN. BHD.,Petaling,Science & Technology,"Mathematics, Statistics & Information Sciences",Full time,"RM 10,000 – RM 14,000 per month","Support the oversight of data science projects to ensure their timely and precise execution, delivering advanced analytics and insights. Address project hurdles collaboratively and refine project outcomes.
+Assist the Data Scientist Manager in mentoring and directing the data science team. Offer guidance on project priorities, technical challenges, and career growth.
+Engage with your team to craft and execute predictive models leveraging sophisticated statistical and machine learning techniques to tackle complex business issues.
+Aid in the exploration and preprocessing of large datasets to unearth valuable features and insights.
+Collaborate effectively with multidisciplinary teams to grasp business goals, align data science projects, and articulate insights to non-technical audiences.
+Drive the enhancement of data science methodologies and processes. Keep abreast of industry standards and evolving trends.
+Holding a minimum of Master's or Ph.D. in Data Science, Statistics, Computer Science, or any other related fields.
+Demonstrable experience as a Data Scientist, including roles in leadership or management.
+High proficiency in languages such as Python or R, along with familiarity with pertinent libraries and frameworks.
+A thorough grasp and application experience of machine learning algorithms in practical scenarios.
+Excellent interpersonal and communication skills for effective teamwork and cross-functional collaboration.
+A commitment to continuous learning, staying updated with data science trends, techniques, and tools, and participating in relevant training and development programs.
+Proven leadership ability to mentor and inspire a team of data scientists.
+Exceptional project management capabilities, with a knack for prioritizing, resource allocation, and deadline adherence.
+Skilled in dissecting large, intricate datasets to extract significant insights.
+Proficiency in utilizing advanced statistical and machine learning methods to address real-world challenges.
+Competent in programming languages like Python or R for data analysis and modeling.
+Strong communication skills, capable of explaining complex findings to a broad audience range.
+How many years' experience do you have as an Assistant Manager?
+Which of the following programming languages are you experienced in?
+Which of the following data analytics tools are you experienced with?
+Which of the following languages are you fluent in?
+How would you rate your Mandarin language skills?"
+20,74488251,Data Engineer/Data Cloud Engineer/Salesforce,LIEGE SOLUTIONS SDN. BHD.,Kuala Lumpur,Information & Communication Technology,Engineering - Software,Full time,"RM 8,000 – RM 12,000 per month","Develop, build, and implement Salesforce Data Cloud instances in conjunction with Salesforces (Sales/Service, Salesforce Marketing Cloud, etc.)
+Develop, oversee, and maintain key data model, segmentation, architecture, code documentation, and technical specifications for designated workstreams.
+Lead the development and physical design of solutions, guide the development team, and ensure the system's architectural integrity.
+Implement Salesforce Data Cloud, emphasizing data modelling, performance optimization, and integrations.
+Function as a technical solution specialist within the Salesforce Data Cloud ecosystem, including components like Data streams, Unified Data Model, Transformations/Insights, Identity Resolution, Segmentations, etc. 
+Drive end-to-end project execution, encompassing analysis, documentation, solution development, testing, and performance, to meet client expectations.
+Identify technical design and data model risks, confirm their validity, and devise strategies to mitigate them.
+Actively participate in technical design and data model workshops alongside Engineering Teams and business stakeholders.
+Clearly translate and communicate technical requirements and solutions to the business using story-based narratives, and present strategies and recommendations to executives.
+Ensure that technical documentation is consistently updated through active participation in technical workshops.
+Lead sessions to understand client needs in technical marketing and Salesforce Data Cloud requirements, and act as a crucial technical resource throughout the implementation process.
+Assess client requirements, propose effective solutions, and communicate the capabilities and limitations of Salesforce Data Cloud, Marketing Cloud Engage, Personalization, Intelligence, Sales, Service, and other relevant SF ecosystem platforms.
+Oversee and coordinate testing activities, including System Integration Testing (SIT) and User Acceptance Testing (UAT), in line with the project's scope.
+Manage workstreams in large-scale enterprise implementations, focusing on Salesforce Data Cloud architecture and data models.
+Bachelor's Degree.
+Experience with at least 1 end-to-end Salesforce Data Cloud implementation.
+Experience with at least 1 Salesforce cross-cloud integrations.
+5+ years in data model and architecture in the Salesforce ecosystem.
+5+ years in consulting, project implementation for global based client, or project management using Agile methodologies.
+5+ years in Customer Data Modelling best practices.
+Proficiency in moderately complex SQL queries, SOQL, Python.
+Experience in integrating with cloud-based data warehouses/data lakes (e.g. Snowflake, Databricks, etc.).
+Effective communication to both technical and non-technical audiences.
+Experience with Tableu, Data Analytic Platform (e.g. PowerBI).
+Experience with Segmentation strategy.
+Experience with Marketing Cloud Personalization.
+Experience with Adobe, Marketo, Oracle or other enterprise marketing automation or CDP platforms.
+Front-end coding skills (HTML, CSS/SASS, Typescript, React, XML, JSON).
+Salesforce certifications (Salesforce Data Cloud Consultant , and Salesforce Certifications - Marketing Cloud, Admin, etc.).
+A supportive, friendly and professional team environment;
+Stability to develop your career with an established and growth focused company; 
+A range of benefits as part of our Employee Benefit Program;
+Which of the following statements best describes your right to work in Malaysia?
+What's your expected monthly basic salary?
+How many years' experience do you have as a Salesforce Developer?
+How many years' experience do you have as a Salesforce Architect?
+How many years' experience do you have as a Salesforce Marketing Cloud Specialist?
+How many years' experience do you have as a Salesforce Engineer?"
+21,74354434,Data Analyst,EMART24 HOLDINGS SDN. BHD.,Petaling,Information & Communication Technology,Business/Systems Analysts,Full time,"RM 6,800 – RM 7,000 per month","Develop and maintain dashboards and reports that provide insights into key performance indicators (KPIs) across all business functions.
+Partner with IT team to leverage technology infrastructure and tools to optimize data collection, storage, and analysis processes.
+Conduct ad-hoc analysis to answer business questions and identify opportunities for growth and improvement.
+Collaborate with cross-functional teams to identify, analyze and recommend data driven solutions to business problems.
+Develop and maintain data pipelines to ensure data is accurate, consistent, and available in a timely manner.
+Identify and recommend BI tools that can help to improve data visibility and analysis across the organization.
+Train and support end-users on the use of BI tools and dashboards.
+Provide recommendations to stakeholders on how to use data to drive growth and improve bottom-line results.
+Continuously monitor the performance of the business and make recommendations for improvement based on data analysis.
+Maintain a high level of data quality, accuracy, and consistency.
+Bachelor's degree in Actuarial Science, Computer Science, Information Technology, or a related field.
+Proven experience (3-5 years) in a business intelligence or data analyst role, preferably in a retail or eCommerce environment.
+Experience using BI tools such as Power BI, Tableau, Metabase, or Google Data Studio.
+Proficient in SQL and/or Python, and data visualization tools.
+Strong analytical and problem-solving skills with the ability to work with complex data sets.
+Ability to work collaboratively and communicate effectively with cross-functional teams.
+Knowledge of retail or eCommerce industry and data trends.
+Excellent communication skills, both written and verbal, with the ability to communicate complex ideas in a simple and clear manner.
+Ability to prioritize and manage multiple projects simultaneously.
+Proficient in both English and Bahasa Malaysia.
+Which of the following statements best describes your right to work in Malaysia?
+What's your expected monthly basic salary?
+How many years' experience do you have as a data analyst?
+Which of the following programming languages are you experienced in?
+How many years' experience do you have using SQL queries?
+Which of the following Relational Database Management Systems (RDBMS) are you experienced with?
+How would you rate your English language skills?
+Which of the following languages are you fluent in?"
+22,74366437,Data Analyst,Asia Careers Group SDN BHD,Kuala Lumpur,Science & Technology,"Mathematics, Statistics & Information Sciences",Full time,"RM 3,700 – RM 5,500 per month","Make large and complex data more 
+accessible, understandable and usable
+ within Tableau
+Transform, improve and integrate
+ graduate outcomes and average salary data
+Combining data result
+ sets across multiple sources using Excel and or other databased platforms
+Maintain and manage existing data collection tools
+, analytics tools and reporting platforms
+Maintain existing reports and develop new report where necessary to support business needs. 
+Delivering the data in a useful and appealing way to clients
+Excellent analytical skills
+Ability to work with large amounts of information and see the ‘bigger picture’
+Comfortable with 
+juggling facts, figures, and number crunching
+Strong communication skills
+, both written and oral and 
+able to work well with other teams
+Proficient in interacting with computer softwares
+Critical thinking
+: able to look at numbers, trends, and data and come to new conclusions based on the findings
+Attention to detail
+Excellent maths skills
+Working to deadlines
+Independent
+Drive data collection
+ and input data collection with 
+preprogrammed scripts
+To
+ collate and merge data sets
+ to enable reporting within Tableau
+Analyse current and past trends
+ of key performance indicators, highlighting trends and analysing causes of unexpected variances
+Develop models and analysis to 
+support strategic initiatives
+Create presentations that provide insightful analysis, 
+identify and propose actions for effective decisions
+Create infographics and generate insight
+ for both clients and for use in marketing and promotion
+To 
+perform ad hoc requests
+ for data on a day-to-day basis
+Minimum 1 years working experience in data analytics and visualisation
+Must be proficient in Power BI, 
+Tableau, Excel
+. Coding skills (Python / NodeJS) and graphic designing skills (Adobe Illustrator) are an advantage but not essential
+Experience creating, integrating dashboards to present data in a meaningful and interactive manner
+Good team worker
+ and able to multitask & meet tight deadlines
+Results oriented and creative
+Required language(s): 
+Proficient in English
+, other foreign languages a plus.
+Full time employment
+Company provided computer
+Exposure to fun and challenging problems in an agile environment
+Transparent company culture and openness to new ideas
+Training courses in your areas of interest
+Which of the following languages are you fluent in?
+Are you willing to undergo a pre-employment background check?
+How much notice are you required to give your current employer?"
+23,74354364,Data Analyst,PERSOLKELLY Workforce Solutions Malaysia Sdn Bhd,Kuala Lumpur,Consulting & Strategy,Analysts,Contract/Temp,"RM 5,500 – RM 6,500 per month","Provide quantitative analytics support to the AML Transformation & Governance team to enhance risk assessment of ongoing current and emerging AML centric risks in core areas; business, client, operations & system controls within the international business entities. 
+Develop and enhance quantitative/qualitative risk assessment model to proactively identify risks enhancement opportunities in areas that are not clearly defined by legislative AML guidance 
+Provide data extraction & quantitative validation for targeted AML risks or ongoing AML issues identified 
+Provide other ad hoc data extraction and validation requests from time to time and assist in the maintenance of data library for all IBG entities 
+Provide periodic quantitative entities’ risk assessment report as part of AML governance update to the AML IBG management 
+Provide collaboration support with the AML data team on entity level business intelligence related request from regional AML teams. 
+Bachelor degree or above majoring in computer science/engineering, statistics, Mathematics, finance and similar fields; Candidate with experience in AML risk modelling and data analytics are preferred 
+At least three years' AML/risk control/fraud management working experience in FI, PI, consulting firms or similar industries 
+Proficient in data handling coding (e.g., SQL) and at least one professional analytics tool such as SAS, python etc. 
+Familiar with at least one popular BI tool, such as Power BI, Tableau.
+Strong ability in logical thinking and problem solving 
+Able to plan & work independently on required deliverables and deadlines 
+Able to work under pressure and result-oriented 
+Highly flexible and detail-oriented with the ability to multi-task and change directions based on demand
+How many years' experience do you have as a data analyst?
+Which of the following languages are you fluent in?
+How many years' experience do you have as a business intelligence (BI) analyst?"
+24,74323206,Senior Data Analyst,AGENSI PEKERJAAN LEWIS TALENT CONSULTING SDN. BHD.,Petaling,Information & Communication Technology,Business/Systems Analysts,Full time,"RM 5,000 – RM 7,000 per month","Lead sophisticated ETL processes to aggregate data from various sources into our data warehouse.
+Perform comprehensive data analysis to uncover trends, patterns, and insights, aiding in addressing business challenges and uncovering opportunities.
+Pioneer the development of complex data visualizations, charts, and dashboards with tools like Tableau and Power BI.
+Ensure data integrity through rigorous quality control, identifying and correcting discrepancies.
+Work closely with different departments to align data insights with business objectives, advising on strategic actions.
+Mentor junior data analysts, promoting a culture of continuous learning and professional growth.
+At least possess a Bachelor’s or Master’s in Data Science, Mathematics, Statistics, Computer Science, or any equivalent fields.
+Technical Proficiency in data analysis (Excel, Python, SQL) and visualization tools (Tableau, Power BI) are required.
+Significant experience in a senior data analyst capacity, evidenced by impactful contributions.
+A solid understanding of Machine Learning, AI, and Data Warehousing principles.
+Leadership Experience: Demonstrated ability to lead and develop a team of data analysts.
+Exceptional analytical skills for dissecting complex datasets and performing advanced statistical analyses to derive actionable insights.
+Demonstrated leadership with the capability to effectively collaborate with both technical and non-technical teams.
+Creative problem-solving skills, offering innovative solutions to business issues.
+Excellent communication, able to simplify complex data findings for a varied audience.
+How many years' experience do you have using SQL queries?
+How would you rate your Mandarin language skills?
+Which of the following languages are you fluent in?
+Which of the following data visualisation tools are you experienced with?
+Which of the following data analytics tools are you experienced with?"
+25,74653378,Data Analyst (E-Commerce),AGENSI PEKERJAAN JS STAFFING SERVICES SDN BHD,Kuala Selangor District,Information & Communication Technology,Database Development & Administration,Full time,"RM 4,000 – RM 4,500 per month","Analyze and interpret key indicators in e-commerce such as sales performance, SKU sales, traffic, conversion rates, buying behavior, pricing, etc. 
+Provide valuable insights and identify trends based on the sales data and suggest recommendations and key action plans 
+Provide competitor analysis 
+Evaluation of the effectiveness of sales and marketing tools and mechanics 
+Creating and developing tools for analysis and information processing 
+Presenting the results of reports and graphical modeling of data 
+Work closely with the E-Commerce sales team to increase sales and revenue 
+Develop and maintain data pipelines to ensure data accuracy, integrity, and availability. 
+Develop and maintain E-Commerce dashboards as per client’s requests. 
+Providing insight to the E-Commerce sales team to optimize marketing expenses. 
+Data Review: reviewing and validating data from various sources (e.g. Lazada, Shopee, etc.) to ensure data quality and accuracy. 
+Candidates must possess at least a Bachelor’s degree in Data Science, Statistics, Computer Science, or a related field 
+Min. 1-2 years of experience as a Data Analyst/Business Analyst/Category Management preferably in E-Commerce or FMCG role 
+Advanced Excel and data mining skills 
+Proficient in data visualization tools (e.g. Power BI) and database query languages (e.g. SQL) 
+Knowledge of Cloud Computing is an added advantage (e.g. Azure) 
+Familiarity with Azure Data Factory and Azure Synapse Pipelines is an added advantage 
+Ability to work independently and take the lead in projects 
+Strong presentation skills 
+Which of the following statements best describes your right to work in Malaysia?
+What's your expected monthly basic salary?"
+26,74552169,Senior Business Intelligence Developer / Data Engineer,CITY FACILITIES MANAGEMENT SDN. BHD.,Kuala Lumpur,Science & Technology,"Mathematics, Statistics & Information Sciences",Full time,"RM 8,000 – RM 12,000 per month","Translation of business requirements into accurate and interactive data visualizations and analytics that enable management and other stakeholders to make informed decisions.
+Drive data mining & data science initiatives
+Review reports to determine basic characteristics, such as origin and report flow, format, frequency, distribution, and purpose or function of report.
+May prepare and issue instructions concerning generation, completion, and distribution of reports according to new or revised practices, procedures, or policies of reports management.
+Examine and evaluate purpose and content of business reports to develop new, or improve existing, format, use, and control.
+Recommend the establishment of new or modified reporting methods and procedures to improve report content and completeness of information.
+Produce associated documentation including functional specifications, testing plans and support documentation.
+Understand and comprehend new system changes as they affect data, reports, and reporting systems.
+Confer with team members originating, handling, processing, or receiving reports to identify problems and to gather suggestions for improvements.
+Manage internal and external stakeholders.
+Provide support and coaching to application support staff and associated team members.
+Provides assistance to support existing infrastructure where appropriate..
+Participate in application validation and QA efforts as they pertain to reporting, data, metrics, and report creation and execution.
+Provides assistance to support existing infrastructure where appropriate.
+Develop the appropriate strategies and plans to ensure a successful implementation that conforms to the company agreed SLA’s.
+A relevant advanced technical degree.
+A mastery of Business Intelligence concepts and the ability to articulate these concepts to technical and non-technical audiences.
+At least 5+ years’ experience in Business Intelligence
+Demonstrable strong data analysis and data mining skills (T-SQL, DAX and MDX).
+Strong data visualisation skills using SSRS & Power BI.
+Exposure to Snowflake on Azure platform and Matillion tools is desirable.
+Demonstrable experience with Business Intelligence tools in the Microsoft BI stack (SSAS and SSIS) or Matillion.
+Strong communication skills with the ability to collaborate and contribute to a team positively.
+Strong analytical skills with the ability to translate business needs into technical solutions.
+The ideal candidate will have experience working in agile projects.
+High cognitive ability to problem solving
+Own and display City values, respect and value others and work as one team. (mandatory all positions)
+Maintain a professional and positive attitude at all times and act with integrity (lead by example).
+High level numerical and analytical skills
+Demonstrated attention to detail and accuracy
+Maintain a high standard of delivery on all work and meet deadlines
+Display high level of motivation and drive at all times.
+Remain calm and organised at all times.
+Ability to develop rapport to ensure effective working relationship with a broad range of managers and team members.
+Demonstrate initiative and confidentiality across the business.
+Treat others with respect and be conscientious and courteous.
+Good time management, planning and organising ability.
+What's your expected monthly basic salary?
+
+
+Role descriptions
+
+
+Salary insights
+
+
+Tools to help you prepare for jobs
+
+"
+27,74976890,Risk Operation Data Analyst ( Online Payment),ALIPAY MALAYSIA SDN. BHD.,Kuala Lumpur,Banking & Financial Services,Analysis & Reporting,Full time,"RM 10,000 – RM 15,000 per month","What's your expected monthly basic salary?
+Which of the following languages are you fluent in?
+How many years' experience do you have as an Operational Risk Analyst?"
+28,74434092,Data Analyst - Internal Audit,Michael Page International (Malaysia) Sdn Bhd,Petaling,"Manufacturing, Transport & Logistics","Warehousing, Storage & Distribution",Full time,$102k - $103k p.a.,"Gain understanding of the company's financial, commercial and operational processes.
+Perform a hybrid role of conducting risk-based financial and operational audits with a core function of data analytics and statistical analysis of existing data sets
+Query and mine large data sets to discover transaction patterns, examine data and filter for targeted information
+Prepare and execute data analysis and interpret the results based on requirements to support the Express audit department and the Data Analytics team
+Prepare data reports and clear data visualizations for stakeholders
+Develop tools, audit approach and audit programs in strong cooperation with the Express audit team to support the internal audit work
+Develop automation for data analysis to increase audit efficiency
+Present key insights using advanced visualization tools, and highlight the data analytics capability to the IA team globally
+Present and discuss findings/data analysis results with management and create high quality audit reports and working papers in line with Internal Audit standards
+Perform requested follow-up procedures as necessary
+Openness to conduct audits across various segments of the company's business, including Operations (OPS), Customs, Finance, and Commercial
+Build and maintain solid relationships with key stakeholders, especially in the business IT area
+Manage own development in the professional area proactively
+Knowledge and experience in MS Office and the following desirable applications: 
+Database query languages (e.g. SQL)
+Power BI, Tableau, Qlik Sense, KNIME
+Programming and coding (e.g. Python, R, C/C++/C#, Visual Basic)
+Cloud computing, Azure, AWS, RPA
+Process mining, e.g. Celonis, DISCO, Signavio
+Excellent English language skills
+Convincing communicator and negotiator
+Analytical and conceptual abilities
+Analytical problem solving and proactive in finding solutions
+High personal responsibility and high degree of integrity
+Ability to work under pressure in a demanding environment
+Rapid comprehension
+Willingness to travel up to 30% of the time (or more if business needs dictate)
+Database query languages (e.g. SQL)
+Power BI, Tableau, Qlik Sense, KNIME
+Programming and coding (e.g. Python, R, C/C++/C#, Visual Basic)
+Cloud computing, Azure, AWS, RPA
+Process mining, e.g. Celonis, DISCO, Signavio
+Minimum two years of experience in data analytics, preferably within the company
+Minimum bachelor's degree in information technology, computer science, engineering, business administration or comparable qualification/background
+First professional experience, preferably within the company
+Attractive remuneration package
+Be part of a globally recognised organisation
+Enhance your analytical skills"

data/01_raw/data_scientist_jobstreet_scraped_v2 2.csv

@@ -1,4 +1,4 @@
-,job_id,job_title,company,descriptions,location,category,subcategory,type,salary
+,job_id,job_title,company,description,location,category,subcategory,type,salary
 0,72761527,Data Engineer,ANHSIN TECHNOLOGY SDN BHD,"Design, develop, and maintain scalable and robust data architecture.
 Create and manage databases, data processing systems, and data integration solutions.
 Develop and implement efficient ETL processes for data ingestion and transformation.

data/02_processed/73981733.json

@@ -0,0 +1 @@
+{"id": 73981733, "location": "kuala lumpur", "company": "private advertiser", "job_title": "big data engineer - cloud", "salary": "RM\u00a09,000 \u2013 RM\u00a013,000 per month", "description": "Which of the following statements best describes your right to work in Malaysia?\nWhat's your expected monthly basic salary?\n\n\nRole descriptions\n\n\nSalary insights\n\n\nTools to help you prepare for jobs\n\n"}

data/02_processed/74323206.json

@@ -0,0 +1 @@
+{"id": 74323206, "location": "petaling", "company": "agensi pekerjaan lewis talent consulting sdn. bhd.", "job_title": "senior data analyst", "salary": "RM\u00a05,000 \u2013 RM\u00a07,000 per month", "description": "Lead sophisticated ETL processes to aggregate data from various sources into our data warehouse.\nPerform comprehensive data analysis to uncover trends, patterns, and insights, aiding in addressing business challenges and uncovering opportunities.\nPioneer the development of complex data visualizations, charts, and dashboards with tools like Tableau and Power BI.\nEnsure data integrity through rigorous quality control, identifying and correcting discrepancies.\nWork closely with different departments to align data insights with business objectives, advising on strategic actions.\nMentor junior data analysts, promoting a culture of continuous learning and professional growth.\nAt least possess a Bachelor\u2019s or Master\u2019s in Data Science, Mathematics, Statistics, Computer Science, or any equivalent fields.\nTechnical Proficiency in data analysis (Excel, Python, SQL) and visualization tools (Tableau, Power BI) are required.\nSignificant experience in a senior data analyst capacity, evidenced by impactful contributions.\nA solid understanding of Machine Learning, AI, and Data Warehousing principles.\nLeadership Experience: Demonstrated ability to lead and develop a team of data analysts.\nExceptional analytical skills for dissecting complex datasets and performing advanced statistical analyses to derive actionable insights.\nDemonstrated leadership with the capability to effectively collaborate with both technical and non-technical teams.\nCreative problem-solving skills, offering innovative solutions to business issues.\nExcellent communication, able to simplify complex data findings for a varied audience.\nHow many years' experience do you have using SQL queries?\nHow would you rate your Mandarin language skills?\nWhich of the following languages are you fluent in?\nWhich of the following data visualisation tools are you experienced with?\nWhich of the following data analytics tools are you experienced with?"}

data/02_processed/74323540.json

@@ -0,0 +1 @@
+{"id": 74323540, "location": "petaling", "company": "agensi pekerjaan lewis talent consulting sdn. bhd.", "job_title": "data scientist", "salary": "RM\u00a06,000 \u2013 RM\u00a08,500 per month", "description": "Understanding the data information, investigating and preprocessing vast datasets to unveil significant features and insights.\nDesign and deploy predictive models employing advanced statistical and machine learning techniques to address complex business issues.\nGenerate compelling visualizations to illustrate complex analyses and insights to both technical and non-technical audiences.\nEngage closely with multidisciplinary teams to grasp business objectives and partake in data-driven projects.\nKeep abreast of the latest data science advancements and enhance the organization's data science methodologies.\nAt least possess a Master's or Ph.D. in Data Science, Statistics, Computer Science, or related fields.\nDemonstrated experience as a Data Scientist, supported by a portfolio of successful data science projects.\nExpertise in Python or R, including familiarity with relevant libraries and frameworks.\nSolid foundation and experience in applying machine learning algorithms to practical problems.\nStrong interpersonal and communication capabilities for effective teamwork with cross-functional groups.\nCommitment to staying informed about industry trends, data science techniques, and tools. Engage in ongoing training and development programs as necessary.\nExpertise in analyzing extensive, complex datasets to extract valuable insights.\nProficiency in applying statistical and machine learning methods to solve real-world challenges.\nCompetence in programming languages like Python or R for data analysis and modeling.\nStrong ability in creating visualizations with tools such as Tableau, Power BI, or similar platforms.\nCreative problem-solving aptitude for tackling complex business issues with data-driven solutions.\nExceptional ability to articulate complex findings and insights clearly.\nHow many years' experience do you have as a data scientist?\nWhich of the following programming languages are you experienced in?\nWhich of the following data analytics tools are you experienced with?\nHow would you rate your Mandarin language skills?\nWhich of the following languages are you fluent in?"}

data/02_processed/74324430.json

@@ -0,0 +1 @@
+{"id": 74324430, "location": "petaling", "company": "agensi pekerjaan lewis talent consulting sdn. bhd.", "job_title": "data scientist assistant manager", "salary": "RM\u00a010,000 \u2013 RM\u00a014,000 per month", "description": "Support the oversight of data science projects to ensure their timely and precise execution, delivering advanced analytics and insights. Address project hurdles collaboratively and refine project outcomes.\nAssist the Data Scientist Manager in mentoring and directing the data science team. Offer guidance on project priorities, technical challenges, and career growth.\nEngage with your team to craft and execute predictive models leveraging sophisticated statistical and machine learning techniques to tackle complex business issues.\nAid in the exploration and preprocessing of large datasets to unearth valuable features and insights.\nCollaborate effectively with multidisciplinary teams to grasp business goals, align data science projects, and articulate insights to non-technical audiences.\nDrive the enhancement of data science methodologies and processes. Keep abreast of industry standards and evolving trends.\nHolding a minimum of Master's or Ph.D. in Data Science, Statistics, Computer Science, or any other related fields.\nDemonstrable experience as a Data Scientist, including roles in leadership or management.\nHigh proficiency in languages such as Python or R, along with familiarity with pertinent libraries and frameworks.\nA thorough grasp and application experience of machine learning algorithms in practical scenarios.\nExcellent interpersonal and communication skills for effective teamwork and cross-functional collaboration.\nA commitment to continuous learning, staying updated with data science trends, techniques, and tools, and participating in relevant training and development programs.\nProven leadership ability to mentor and inspire a team of data scientists.\nExceptional project management capabilities, with a knack for prioritizing, resource allocation, and deadline adherence.\nSkilled in dissecting large, intricate datasets to extract significant insights.\nProficiency in utilizing advanced statistical and machine learning methods to address real-world challenges.\nCompetent in programming languages like Python or R for data analysis and modeling.\nStrong communication skills, capable of explaining complex findings to a broad audience range.\nHow many years' experience do you have as an Assistant Manager?\nWhich of the following programming languages are you experienced in?\nWhich of the following data analytics tools are you experienced with?\nWhich of the following languages are you fluent in?\nHow would you rate your Mandarin language skills?"}

data/02_processed/74354364.json

@@ -0,0 +1 @@
+{"id": 74354364, "location": "kuala lumpur", "company": "persolkelly workforce solutions malaysia sdn bhd", "job_title": "data analyst", "salary": "RM\u00a05,500 \u2013 RM\u00a06,500 per month", "description": "Provide quantitative analytics support to the AML Transformation & Governance team to enhance risk assessment of ongoing current and emerging AML centric risks in core areas; business, client, operations & system controls within the international business entities.\u00a0\nDevelop and enhance quantitative/qualitative risk assessment model to proactively identify risks enhancement opportunities in areas that are not clearly defined by legislative AML guidance\u00a0\nProvide data extraction & quantitative validation for targeted AML risks or ongoing AML issues identified\u00a0\nProvide other ad hoc data extraction and validation requests from time to time and assist in the maintenance of data library for all IBG entities\u00a0\nProvide periodic quantitative entities\u2019 risk assessment report as part of AML governance update to the AML IBG management\u00a0\nProvide collaboration support with the AML data team on entity level business intelligence related request from regional AML teams.\u00a0\nBachelor degree or above majoring in computer science/engineering, statistics, Mathematics, finance and similar fields; Candidate with experience in AML risk modelling and data analytics are preferred\u00a0\nAt least three years' AML/risk control/fraud management working experience in FI, PI, consulting firms or similar industries\u00a0\nProficient in data handling coding (e.g., SQL) and at least one professional analytics tool such as SAS, python etc.\u00a0\nFamiliar with at least one popular BI tool, such as Power BI, Tableau.\nStrong ability in logical thinking and problem solving\u00a0\nAble to plan & work independently on required deliverables and deadlines\u00a0\nAble to work under pressure and result-oriented\u00a0\nHighly flexible and detail-oriented with the ability to multi-task and change directions based on demand\nHow many years' experience do you have as a data analyst?\nWhich of the following languages are you fluent in?\nHow many years' experience do you have as a business intelligence (BI) analyst?"}

data/02_processed/74354434.json

@@ -0,0 +1 @@
+{"id": 74354434, "location": "petaling", "company": "emart24 holdings sdn. bhd.", "job_title": "data analyst", "salary": "RM\u00a06,800 \u2013 RM\u00a07,000 per month", "description": "Develop and maintain dashboards and reports that provide insights into key performance indicators (KPIs) across all business functions.\nPartner with IT team to leverage technology infrastructure and tools to optimize data collection, storage, and analysis processes.\nConduct ad-hoc analysis to answer business questions and identify opportunities for growth and improvement.\nCollaborate with cross-functional teams to identify, analyze and recommend data driven solutions to business problems.\nDevelop and maintain data pipelines to ensure data is accurate, consistent, and available in a timely manner.\nIdentify and recommend BI tools that can help to improve data visibility and analysis across the organization.\nTrain and support end-users on the use of BI tools and dashboards.\nProvide recommendations to stakeholders on how to use data to drive growth and improve bottom-line results.\nContinuously monitor the performance of the business and make recommendations for improvement based on data analysis.\nMaintain a high level of data quality, accuracy, and consistency.\nBachelor's degree in Actuarial Science, Computer Science, Information Technology, or a related field.\nProven experience (3-5 years) in a business intelligence or data analyst role, preferably in a retail or eCommerce environment.\nExperience using BI tools such as Power BI, Tableau, Metabase, or Google Data Studio.\nProficient in SQL and/or Python, and data visualization tools.\nStrong analytical and problem-solving skills with the ability to work with complex data sets.\nAbility to work collaboratively and communicate effectively with cross-functional teams.\nKnowledge of retail or eCommerce industry and data trends.\nExcellent communication skills, both written and verbal, with the ability to communicate complex ideas in a simple and clear manner.\nAbility to prioritize and manage multiple projects simultaneously.\nProficient in both English and Bahasa Malaysia.\nWhich of the following statements best describes your right to work in Malaysia?\nWhat's your expected monthly basic salary?\nHow many years' experience do you have as a data analyst?\nWhich of the following programming languages are you experienced in?\nHow many years' experience do you have using SQL queries?\nWhich of the following Relational Database Management Systems (RDBMS) are you experienced with?\nHow would you rate your English language skills?\nWhich of the following languages are you fluent in?"}

data/02_processed/74360323.json

@@ -0,0 +1 @@
+{"id": 74360323, "location": "kuala lumpur", "company": "talentbank sdn bhd", "job_title": "data scientist", "salary": "RM\u00a06,800 \u2013 RM\u00a010,200 per month", "description": "Which of the following programming languages are you experienced in?\nWhat's your expected monthly basic salary?\nWhich of the following data analytics tools are you experienced with?\nHow many years' experience do you have as a data scientist?"}

data/02_processed/74366437.json

@@ -0,0 +1 @@
+{"id": 74366437, "location": "kuala lumpur", "company": "asia careers group sdn bhd", "job_title": "data analyst", "salary": "RM\u00a03,700 \u2013 RM\u00a05,500 per month", "description": "Make large and complex data more \naccessible, understandable and usable\n within Tableau\nTransform, improve and integrate\n graduate outcomes and average salary data\nCombining data result\n sets across multiple sources using Excel and or other databased platforms\nMaintain and manage existing data collection tools\n, analytics tools and reporting platforms\nMaintain existing reports and develop new report where necessary to support business needs. \nDelivering the data in a useful and appealing way to clients\nExcellent analytical skills\nAbility to work with large amounts of information and see the \u2018bigger picture\u2019\nComfortable with \njuggling facts, figures, and number crunching\nStrong communication skills\n, both written and oral and \nable to work well with other teams\nProficient in interacting with computer softwares\nCritical thinking\n: able to look at numbers, trends, and data and come to new conclusions based on the findings\nAttention to detail\nExcellent maths skills\nWorking to deadlines\nIndependent\nDrive data collection\n and input data collection with \npreprogrammed scripts\nTo\n collate and merge data sets\n to enable reporting within Tableau\nAnalyse current and past trends\n of key performance indicators, highlighting trends and analysing causes of unexpected variances\nDevelop models and analysis to \nsupport strategic initiatives\nCreate presentations that provide insightful analysis, \nidentify and propose actions for effective decisions\nCreate infographics and generate insight\n for both clients and for use in marketing and promotion\nTo \nperform ad hoc requests\n for data on a day-to-day basis\nMinimum 1 years working experience in data analytics and visualisation\nMust be proficient in Power BI, \nTableau, Excel\n. Coding skills (Python / NodeJS) and graphic designing skills (Adobe Illustrator) are an advantage but not essential\nExperience creating, integrating dashboards to present data in a meaningful and interactive manner\nGood team worker\n and able to multitask & meet tight deadlines\nResults oriented and creative\nRequired language(s): \nProficient in English\n, other foreign languages a plus.\nFull time employment\nCompany provided computer\nExposure to fun and challenging problems in an agile environment\nTransparent company culture and openness to new ideas\nTraining courses in your areas of interest\nWhich of the following languages are you fluent in?\nAre you willing to undergo a pre-employment background check?\nHow much notice are you required to give your current employer?"}

data/02_processed/74433737.json

@@ -0,0 +1 @@
+{"id": 74433737, "location": "selangor", "company": "michael page international (malaysia) sdn bhd", "job_title": "data engineer", "salary": "$96k - $97k p.a.", "description": "Attractive remuneration package\nBe part of a globally recognised organisation\nEnhance your development skills"}

data/02_processed/74434092.json

@@ -0,0 +1 @@
+{"id": 74434092, "location": "petaling", "company": "michael page international (malaysia) sdn bhd", "job_title": "data analyst - internal audit", "salary": "$102k - $103k p.a.", "description": "Gain understanding of the company's financial, commercial and operational processes.\nPerform a hybrid role of conducting risk-based financial and operational audits with a core function of data analytics and statistical analysis of existing data sets\nQuery and mine large data sets to discover transaction patterns, examine data and filter for targeted information\nPrepare and execute data analysis and interpret the results based on requirements to support the Express audit department and the Data Analytics team\nPrepare data reports and clear data visualizations for stakeholders\nDevelop tools, audit approach and audit programs in strong cooperation with the Express audit team to support the internal audit work\nDevelop automation for data analysis to increase audit efficiency\nPresent key insights using advanced visualization tools, and highlight the data analytics capability to the IA team globally\nPresent and discuss findings/data analysis results with management and create high quality audit reports and working papers in line with Internal Audit standards\nPerform requested follow-up procedures as necessary\nOpenness to conduct audits across various segments of the company's business, including Operations (OPS), Customs, Finance, and Commercial\nBuild and maintain solid relationships with key stakeholders, especially in the business IT area\nManage own development in the professional area proactively\nKnowledge and experience in MS Office and the following desirable applications: \nDatabase query languages (e.g. SQL)\nPower BI, Tableau, Qlik Sense, KNIME\nProgramming and coding (e.g. Python, R, C/C++/C#, Visual Basic)\nCloud computing, Azure, AWS, RPA\nProcess mining, e.g. Celonis, DISCO, Signavio\nExcellent English language skills\nConvincing communicator and negotiator\nAnalytical and conceptual abilities\nAnalytical problem solving and proactive in finding solutions\nHigh personal responsibility and high degree of integrity\nAbility to work under pressure in a demanding environment\nRapid comprehension\nWillingness to travel up to 30% of the time (or more if business needs dictate)\nDatabase query languages (e.g. SQL)\nPower BI, Tableau, Qlik Sense, KNIME\nProgramming and coding (e.g. Python, R, C/C++/C#, Visual Basic)\nCloud computing, Azure, AWS, RPA\nProcess mining, e.g. Celonis, DISCO, Signavio\nMinimum two years of experience in data analytics, preferably within the company\nMinimum bachelor's degree in information technology, computer science, engineering, business administration or comparable qualification/background\nFirst professional experience, preferably within the company\nAttractive remuneration package\nBe part of a globally recognised organisation\nEnhance your analytical skills"}

data/02_processed/74488251.json

@@ -0,0 +1 @@
+{"id": 74488251, "location": "kuala lumpur", "company": "liege solutions sdn. bhd.", "job_title": "data engineer/data cloud engineer/salesforce", "salary": "RM\u00a08,000 \u2013 RM\u00a012,000 per month", "description": "Develop, build, and implement Salesforce Data Cloud instances in conjunction with Salesforces (Sales/Service, Salesforce Marketing Cloud, etc.)\nDevelop, oversee, and maintain key data model, segmentation, architecture, code documentation, and technical specifications for designated workstreams.\nLead the development and physical design of solutions, guide the development team, and ensure the system's architectural integrity.\nImplement Salesforce Data Cloud, emphasizing data modelling, performance optimization, and integrations.\nFunction as a technical solution specialist within the Salesforce Data Cloud ecosystem, including components like Data streams, Unified Data Model, Transformations/Insights, Identity Resolution, Segmentations, etc.\u00a0\nDrive end-to-end project execution, encompassing analysis, documentation, solution development, testing, and performance, to meet client expectations.\nIdentify technical design and data model risks, confirm their validity, and devise strategies to mitigate them.\nActively participate in technical design and data model workshops alongside Engineering Teams and business stakeholders.\nClearly translate and communicate technical requirements and solutions to the business using story-based narratives, and present strategies and recommendations to executives.\nEnsure that technical documentation is consistently updated through active participation in technical workshops.\nLead sessions to understand client needs in technical marketing and Salesforce Data Cloud requirements, and act as a crucial technical resource throughout the implementation process.\nAssess client requirements, propose effective solutions, and communicate the capabilities and limitations of Salesforce Data Cloud, Marketing Cloud Engage, Personalization, Intelligence, Sales, Service, and other relevant SF ecosystem platforms.\nOversee and coordinate testing activities, including System Integration Testing (SIT) and User Acceptance Testing (UAT), in line with the project's scope.\nManage workstreams in large-scale enterprise implementations, focusing on Salesforce Data Cloud architecture and data models.\nBachelor's Degree.\nExperience with at least 1 end-to-end Salesforce Data Cloud implementation.\nExperience with at least 1 Salesforce cross-cloud integrations.\n5+ years in data model and architecture in the Salesforce ecosystem.\n5+ years in consulting, project implementation for global based client, or project management using Agile methodologies.\n5+ years in Customer Data Modelling best practices.\nProficiency in moderately complex SQL queries, SOQL, Python.\nExperience in integrating with cloud-based data warehouses/data lakes (e.g. Snowflake, Databricks, etc.).\nEffective communication to both technical and non-technical audiences.\nExperience with Tableu, Data Analytic Platform (e.g. PowerBI).\nExperience with Segmentation strategy.\nExperience with Marketing Cloud Personalization.\nExperience with Adobe, Marketo, Oracle or other enterprise marketing automation or CDP platforms.\nFront-end coding skills (HTML, CSS/SASS, Typescript, React, XML, JSON).\nSalesforce certifications (Salesforce Data Cloud Consultant , and Salesforce Certifications - Marketing Cloud, Admin, etc.).\nA supportive, friendly and professional team environment;\nStability to develop your career with an established and growth focused company;\u00a0\nA range of benefits as part of our Employee Benefit Program;\nWhich of the following statements best describes your right to work in Malaysia?\nWhat's your expected monthly basic salary?\nHow many years' experience do you have as a Salesforce Developer?\nHow many years' experience do you have as a Salesforce Architect?\nHow many years' experience do you have as a Salesforce Marketing Cloud Specialist?\nHow many years' experience do you have as a Salesforce Engineer?"}

data/02_processed/74513015.json

@@ -0,0 +1 @@
+{"id": 74513015, "location": "petaling", "company": "ambition group malaysia sdn bhd", "job_title": "senior data scientist (full stack)", "salary": "Attractive Remuneration Package", "description": "Utilize technical proficiency for data gathering, cleaning, and preprocessing from varied sources.\nDevelop, train, and assess machine learning models for solving business problems like predictive analytics and natural language processing.\nConduct statistical analysis, hypothesis testing, and data-driven experimentation; stay updated on emerging trends in data science and machine learning\nDegree/Master in Computer Science, Mathematics, Statistics.\nExperienced in all aspects of the data science lifecycle, including problem formulation, data collection and preprocessing, exploratory data analysis, feature engineering, model selection and training, model evaluation.\nFamiliar working with cloud-based big data platforms (Azure, GCP, AWS)\nA minimum 3 years of hands on experience with Python.\nExcellent communication skills in Bahasa Melayu and English."}

data/02_processed/74552169.json

@@ -0,0 +1 @@
+{"id": 74552169, "location": "kuala lumpur", "company": "city facilities management sdn. bhd.", "job_title": "senior business intelligence developer / data engineer", "salary": "RM\u00a08,000 \u2013 RM\u00a012,000 per month", "description": "Translation of business requirements into accurate and interactive data visualizations and analytics that enable management and other stakeholders to make informed decisions.\nDrive data mining & data science initiatives\nReview reports to determine basic characteristics, such as origin and report flow, format, frequency, distribution, and purpose or function of report.\nMay prepare and issue instructions concerning generation, completion, and distribution of reports according to new or revised practices, procedures, or policies of reports management.\nExamine and evaluate purpose and content of business reports to develop new, or improve existing, format, use, and control.\nRecommend the establishment of new or modified reporting methods and procedures to improve report content and completeness of information.\nProduce associated documentation including functional specifications, testing plans and support documentation.\nUnderstand and comprehend new system changes as they affect data, reports, and reporting systems.\nConfer with team members originating, handling, processing, or receiving reports to identify problems and to gather suggestions for improvements.\nManage internal and external stakeholders.\nProvide support and coaching to application support staff and associated team members.\nProvides assistance to support existing infrastructure where appropriate..\nParticipate in application validation and QA efforts as they pertain to reporting, data, metrics, and report creation and execution.\nProvides assistance to support existing infrastructure where appropriate.\nDevelop the appropriate strategies and plans to ensure a successful implementation that conforms to the company agreed SLA\u2019s.\nA relevant advanced technical degree.\nA mastery of Business Intelligence concepts and the ability to articulate these concepts to technical and non-technical audiences.\nAt least 5+ years\u2019 experience in Business Intelligence\nDemonstrable strong data analysis and data mining skills (T-SQL, DAX and MDX).\nStrong data visualisation skills using SSRS & Power BI.\nExposure to Snowflake on Azure platform and Matillion tools is desirable.\nDemonstrable experience with Business Intelligence tools in the Microsoft BI stack (SSAS and SSIS) or Matillion.\nStrong communication skills with the ability to collaborate and contribute to a team positively.\nStrong analytical skills with the ability to translate business needs into technical solutions.\nThe ideal candidate will have experience working in agile projects.\nHigh cognitive ability to problem solving\nOwn and display City values, respect and value others and work as one team. (mandatory all positions)\nMaintain a professional and positive attitude at all times and act with integrity (lead by example).\nHigh level numerical and analytical skills\nDemonstrated attention to detail and accuracy\nMaintain a high standard of delivery on all work and meet deadlines\nDisplay high level of motivation and drive at all times.\nRemain calm and organised at all times.\nAbility to develop rapport to ensure effective working relationship with a broad range of managers and team members.\nDemonstrate initiative and confidentiality across the business.\nTreat others with respect and be conscientious and courteous.\nGood time management, planning and organising ability.\nWhat's your expected monthly basic salary?\n\n\nRole descriptions\n\n\nSalary insights\n\n\nTools to help you prepare for jobs\n\n"}

data/02_processed/74586835.json

@@ -0,0 +1 @@
+{"id": 74586835, "location": "kuala lumpur", "company": "michael page international (malaysia) sdn bhd", "job_title": "data engineer", "salary": "$36k - $64k p.a. + Bonus + Other benefits", "description": "Designing and implementing Data pipelines and ETL processes from various sources into our Data Warehouse\nEnsure that data is cleaned, standardized, and properly structured to minimize data inconsistencies and gaps\nCreate and maintain data pipelines to facilitate the smooth flow of data throughout the organization\nWork closely with cross-functional teams to identify data integration needs and implement solutions that enable seamless data movement\nEstablish data quality standards and processes to monitor and maintain the accuracy and integrity of data within Big Query and other data systems\nImplement data validation techniques to identify and rectify invalid or incomplete data entries\nDesign and maintain data schemas to ensure consistency and efficiency in data storage and retrieval\nCollaborate with Tech and Business teams to refine data models and adapt to changing business needs\nDrive data-centric decision-making within the organization\nCollaborate with stakeholders to foster a data-driven culture by promoting data literacy and usage\nBachelor's degree in Computer Science, Engineering or a related field\nProven experience as a Data Engineer or similar role, with a track record of implementing successful data solutions\nProficiency in ETL (Extract, Transform, Load) processes and data integration techniques\nStrong knowledge of database systems and data warehousing concepts\nExperience with Google Cloud Platform (GCP) services"}

data/02_processed/74601731.json

@@ -0,0 +1 @@
+{"id": 74601731, "location": "kuala lumpur", "company": "air energy consulting (malaysia) sdn bhd", "job_title": "data analyst internship", "salary": "RM\u00a0900 \u2013 RM\u00a01,000 per month", "description": "Review and update on Bullhorn data for Asia placements.\nWork with the Service Team Australia on updating the Bullhorn data for data accuracy.\nWork closely with the Quality Coordinator on the accuracy and completeness of the data (Client\u2019s PO/WO and Contractors\u2019 employment packages) within the Airswift systems.\nWork with the relevant parties to ensure that any errors are corrected promptly.\nAdministrator and investigate client, assignee and consultant surveys.\nData analysis \u2013 accuracy report that was generated for the entire Asia Pacific Region.\nConsolidate data and dashboards for distribution to SVPs and Global Operations as required.\nManage the Business Continuity plan and Office Safety for the super region and specific country.\nWork with other members of the Regional Commercial Team, ensuring that all requests for review, questions and queries are addressed, and escalating to the relevant people when required.\nThe entire process flow of how a manpower/workforce solution provider organisation functions (Sales/BD -> Recruitment -> Onboarding -> Payroll -> Billing).\nDiploma or Degree in Business Administration or Management or equivalent.\nKnowledge of relevant regulatory requirements in the region.\nExcellent technical skills including all Microsoft Office products and understanding of statistics.\nAbility to problem solve with high attention to detail.\nExcellent written and verbal communication and interpersonal skills.\nMust have a sense of urgency in completing tasks.\nClient service orientated with good teamwork skills.\nOwnership and willingness to accept responsibility.\nVirtual Onboarding Events.\nTeam-driven environment fostering a supportive culture with a strong emphasis on work-life balance.\nCheers! where your peers can nominate you for outstanding performance.\nCoaching and Training are provided.\nGrowth\n - In life and business, one must grow to flourish and achieve high ambitions. Growth requires change, challenge, risk and sacrifice - we will always choose growth.\nSafety\n - We create a safe and inclusive environment to protect the physical and psychological well-being of everyone we work with.\nExcellence\n - We deliver, holding ourselves accountable for results. Our customers see excellence in everything we do.\nIntegrity\n - We are ethical, open, honest and authentic. People trust us to do the right thing for the right reason.\nWhich of the following statements best describes your right to work in Malaysia?"}

data/02_processed/74602585.json

@@ -0,0 +1 @@
+{"id": 74602585, "location": "kulim district", "company": "persolkelly workforce solutions malaysia sdn bhd", "job_title": "data analyst/scientist", "salary": "RM\u00a03,500 \u2013 RM\u00a05,000 per month", "description": "What's your expected monthly basic salary?\nHow many years' experience do you have as a data analyst?\nHow many years' experience do you have using SQL queries?"}

data/02_processed/74636692.json

@@ -0,0 +1 @@
+{"id": 74636692, "location": "kuala selangor district", "company": "agensi pekerjaan talent recruit sdn. bhd.", "job_title": "data analyst", "salary": "RM\u00a04,000 \u2013 RM\u00a05,000 per month", "description": "Analyze and interpret key indicators in e-commerce\nProvide valuable insights and identify trends based on the sales data and suggest recommendations and key action plans\u00a0\nProvide competitor analysis\u00a0\nEvaluation of the effectiveness of sales and marketing tools and mechanics\nCreating and developing tools for analysis and information processing\u00a0\nPresenting the results of reports and graphical modeling of data\u00a0\nWork closely with the e-Commerce sales team to increase sales and revenue\u00a0\nDevelop and maintain data pipelines to ensure data accuracy, integrity, and availability\nDevelop and maintain e-Commerce dashboards as per client\u2019s requests\nProviding insight to the e-Commerce sales team to optimize marketing expenses\nData Review: reviewing and validating data from various sources (e.g. Lazada, Shopee, etc.) to ensure data quality and accuracy\nDiploma or Bachelor\u2019s degree in Data Science, Statistics, Computer Science, or a related field\u00a0\nMin \n1 - 2 years of experience as a Data Analyst/Business Analyst/Category Management preferably in E-Commerce or FMCG industry\nAdvanced Excel and data mining skills\u00a0\nProficient in data visualization tools (e.g. Power BI) and database query languages (e.g. SQL)\u00a0\nKnowledge of Cloud Computing is an added advantage (e.g. Azure)\u00a0\nFamiliarity with Azure Data Factory and Azure Synapse Pipelines is an added advantage\u00a0\nAbility to work independently and take the lead in projects\u00a0\nStrong presentation skills\nPermanent employment\nWorking hours : Mon - Fri (9am \u2013 6pm) *\nSaturday half day only when required by Management during certain peak events (very occasional)\nWorking location : Puncak Alam\nWhat's your expected monthly basic salary?"}

data/02_processed/74653378.json

@@ -0,0 +1 @@
+{"id": 74653378, "location": "kuala selangor district", "company": "agensi pekerjaan js staffing services sdn bhd", "job_title": "data analyst (e-commerce)", "salary": "RM\u00a04,000 \u2013 RM\u00a04,500 per month", "description": "Analyze and interpret key indicators in e-commerce such as sales performance, SKU sales, traffic, conversion rates, buying behavior, pricing, etc.\u00a0\nProvide valuable insights and identify trends based on the sales data and suggest recommendations and key action plans\u00a0\nProvide competitor analysis\u00a0\nEvaluation of the effectiveness of sales and marketing tools and mechanics\u00a0\nCreating and developing tools for analysis and information processing\u00a0\nPresenting the results of reports and graphical modeling of data\u00a0\nWork closely with the E-Commerce sales team to increase sales and revenue\u00a0\nDevelop and maintain data pipelines to ensure data accuracy, integrity, and availability.\u00a0\nDevelop and maintain E-Commerce dashboards as per client\u2019s requests.\u00a0\nProviding insight to the E-Commerce sales team to optimize marketing expenses.\u00a0\nData Review: reviewing and validating data from various sources (e.g. Lazada, Shopee, etc.) to ensure data quality and accuracy.\u00a0\nCandidates must possess at least a Bachelor\u2019s degree in Data Science, Statistics, Computer Science, or a related field\u00a0\nMin. 1-2 years of experience as a Data Analyst/Business Analyst/Category Management preferably in E-Commerce or FMCG role\u00a0\nAdvanced Excel and data mining skills\u00a0\nProficient in data visualization tools (e.g. Power BI) and database query languages (e.g. SQL)\u00a0\nKnowledge of Cloud Computing is an added advantage (e.g. Azure)\u00a0\nFamiliarity with Azure Data Factory and Azure Synapse Pipelines is an added advantage\u00a0\nAbility to work independently and take the lead in projects\u00a0\nStrong presentation skills\u00a0\nWhich of the following statements best describes your right to work in Malaysia?\nWhat's your expected monthly basic salary?"}

data/02_processed/74703677.json

@@ -0,0 +1 @@
+{"id": 74703677, "location": "bangsar south", "company": "private advertiser", "job_title": "junior data analyst", "salary": "RM\u00a03,500 \u2013 RM\u00a04,300 per month", "description": "Analyze and manage data related to E-sports events, focusing on Key Performance Indicators (KPIs).\nCreate daily reports highlighting discrepancies or errors identified during the analysis process.\nChart and conduct in-depth analyses of team performance in E-sports events.\nReal-time monitoring and analysis of data, including the production of charts and performance evaluations.\nMaintain daily event reports with important metrics for overall performance.\nCollaborate with team members, ensuring smooth operations and providing support.\nFluency in Mandarin preferred for client interaction.\nStrong analytical and critical thinking skills.\nAbility to work independently under pressure and manage multiple projects with tight schedules.\nFlexibility for shifts and overnight work.\nPassion for E-sports is a plus.\nFresh graduates are welcome to apply.\n13 months\u2019 Salary\nQuarterly and annual bonuses\nMonthly meal, shift, and parking allowances\nMonthly team gatherings & birthday celebrations\nUnlimited snacks & beverages\nTeam building activities & outings\nMedical insurance\n14 days of annual leave\nBirthday leave\n8 hours of work per day\nEPF, SOCSO, EIS contributions\nWhich of the following languages are you fluent in?"}

data/02_processed/74727704.json

@@ -0,0 +1 @@
+{"id": 74727704, "location": "kuala lumpur", "company": "private advertiser", "job_title": "machine learning engineer (artificial intelligence)", "salary": "RM\u00a04,000 \u2013 RM\u00a06,000 per month", "description": "Design, develop, and deploy machine learning models and algorithms for complex and unique datasets, using various techniques such as mathematical modeling, scikit-learn, NLP, CNN, RNN, DL, RL, Transformers, GAN, LLM, RAG\nCollaborate with cross-functional teams to extract insights, identify business opportunities and provide data-driven recommendations\nStay up-to-date with the latest machine learning and AI techniques and tools\nCommunicate complex technical concepts to non-technical stakeholders in an easy-to-understand manner\nBachelor's degree or higher in Computer Science, Mathematics, Statistics, Actuarial Science, Informatics, Information Science or related fields\nStrong analytical skills and attention to detail\nParticipation in Kaggle, Mathematics Olympiad or similar competitions is a plus\nExcellent programming skills in Python, R, Java, or C++\nFamiliar with ML frameworks such as Tensorflow, Keras, PyTorch, MLFlow, AutoML, TensorRT, CUDA\nExcellent communication and collaboration skills\nExperience with designing, training, and deploying machine learning models\nCustomer centric and committed to deliver the best AI results to customers\nWhat's your expected monthly basic salary?\n\n\nRole descriptions\n\n\nSalary insights\n\n\nTools to help you prepare for jobs\n\n"}

data/02_processed/74789886.json

@@ -0,0 +1 @@
+{"id": 74789886, "location": "petaling", "company": "oriental fortune trading malaysia sdn bhd", "job_title": "junior data analyst", "salary": "RM\u00a03,000 \u2013 RM\u00a04,300 per month", "description": "Data Collection and Cleaning:\n Collect and compile data from various sources, ensuring data accuracy and completeness. Perform data cleaning and transformation as necessary to prepare it for analysis.\nData Analysis:\n Assist in conducting data analysis using statistical methods, data visualization tools, and data analysis software. Summarize findings in clear and concise reports.\nData Visualization:\n Create visually appealing and informative charts, graphs, and dashboards to present data insights to stakeholders. Use tools such as Tableau, Power BI, or Excel for visualization.\nReport Generation:\n Prepare regular and ad-hoc reports to communicate key metrics and findings to the team and management. Ensure reports are accurate and delivered on time.\nData Quality Assurance:\n Help maintain data quality standards by identifying and resolving data discrepancies or inconsistencies. Collaborate with other team members to establish data quality protocols.\nCollaboration: \nWork closely with cross-functional teams to understand data requirements and deliver actionable insights.\nResearch and Learning:\n Stay up-to-date with industry trends and best practices in data analysis. Continuously expand your knowledge and skills in data analytics techniques and tools.\nBachelor's degree in a relevant field such as Finance,\u00a0Data Science, Statistics, Mathematics, Computer Science, or a related discipline.\nCandidate with 1-2 years quantitative analysis or finance background are preferred; \nfresh graduates are welcome as well.\nRequired language(s): \nFluent in English & Mandarin\n are preferred because the role requires candidates to interact with Mandarin-speaking clients.\nStrong analytical skills and a passion for working with data.\nProficiency in data analysis tools and programming languages such as Excel, SQL, Python, or R.\nFamiliarity with data visualization tools (e.g., Tableau, Power BI) is a plus.\nExcellent communication skills, with the ability to translate complex data into understandable insights.\nDetail-oriented and capable of maintaining data accuracy and integrity.\nHighly organized and able to manage multiple tasks and deadlines effectively.\nTeam player with a collaborative mindset and the ability to work well in a diverse team environment.\nEagerness to learn and adapt to new tools, techniques, and technologies in the field of data analysis.\nPossess own car.\nWhat's your expected monthly basic salary?\nHow many years' experience do you have as a data analyst?\nWhich of the following languages are you fluent in?"}

data/02_processed/74850440.json

@@ -0,0 +1 @@
+{"id": 74850440, "location": "kuala lumpur", "company": "phenomenal uam sdn bhd", "job_title": "data scientist", "salary": "RM\u00a05,000 \u2013 RM\u00a07,500 per month", "description": "Utilizing machine learning techniques to analyze and interpret complex data sets.\nDeveloping AI models and algorithms to solve specific business problems and improve operational efficiency.\nCollaborating with engineers and software developers to integrate AI solutions into our products and services.\nConducting exploratory data analysis to identify trends, patterns, and insights.\nDesigning and implementing experiments to validate model performance and accuracy.\nCommunicating findings and insights to stakeholders through visualizations, presentations, and reports.\nStaying abreast of the latest advancements in AI and machine learning technologies, and evaluating their potential applications within our organization.\nBachelor's degree in Computer Science, Data Science, Statistics, Mathematics, or a related field. Advanced degree (Master's or Ph.D.) preferred.\nProven experience (X years) working as a Data Scientist or AI Engineer, with a strong focus on developing and implementing machine learning algorithms and AI models.\nProficiency in programming languages such as Python, R, or Java, along with experience using libraries and frameworks such as TensorFlow, PyTorch, or scikit-learn.\nSolid understanding of statistical analysis, hypothesis testing, and experimental design.\nExperience with data visualization tools such as Matplotlib, Seaborn, or Tableau.\nStrong problem-solving skills and the ability to translate business requirements into analytical solutions.\nExcellent communication and collaboration skills, with the ability to effectively convey technical concepts to both technical and non-technical stakeholders.\nFamiliarity with cloud computing platforms (e.g., AWS, Azure, Google Cloud) and big data technologies (e.g., Hadoop, Spark) is a plus.\nPassion for innovation and a keen interest in staying updated with the latest trends and advancements in AI and machine learning."}

data/02_processed/74866336.json

@@ -0,0 +1 @@
+{"id": 74866336, "location": "petaling", "company": "elken sdn bhd", "job_title": "manager - data engineer", "salary": "RM\u00a08,000 \u2013 RM\u00a012,000 per month", "description": "Provide database operation support including monitoring, performance tuning, backup and restoration, disaster recovery, troubleshooting, replication setup and capacity management.\nResponsible for installing, configuration, upgrading and maintaining multiple environments including on premise and cloud databases.\nProvide recommendation for future direction of database technology and infrastructure setup.\nDeploy database objects and data patches when required by development team.\nAssist developers with complex query tuning and best practice suggestion.\nDatabase environment setup for Development, UAT and Production.\nDatabase migration for various platform.\nBuild automation script for database tasks.\nCollaborate with Data Analysts, and other stakeholders to understand data requirements and provide data solutions to meet their needs.\nBachelor\u2019s Degree in Computer Science or equivalent in related field.\nAt least 3 years working experience in related field.\nPossess professional database certifications such as Microsoft Certified, Oracle Certified will be added advantage.\nExperience with RDBMS Administration such as MS SQL, My SQL MariaDB, PostgresSQL and Oracle.\nExperience with AWS RDS Cloud database environment administration.\nExperience with Windows Server and Linux environments.\nExperience with Powershell scripting, Linux command line.\nAble to converse in\u00a0\nMandarin\n\u00a0as needed to support oversea team like Hong Kong, China and Taiwan.\nWhat's your expected monthly basic salary?\nWhich of the following Relational Database Management Systems (RDBMS) are you experienced with?"}

data/02_processed/74879920.json

@@ -0,0 +1 @@
+{"id": 74879920, "location": "sri hartamas", "company": "mont royale college", "job_title": "data analyst (intern)", "salary": "RM\u00a0600 per month", "description": "Collect, organize, and analyze data from multiple sources to identify trends, patterns, and insights.\nDevelop and maintain databases and data systems necessary for data analysis and reporting.\nGenerate regular reports and dashboards to communicate key findings and metrics to stakeholders.\nCollaborate with departments to understand their data needs and provide analytical support to inform strategic initiatives.\nConduct thorough data quality assessments and implement solutions to address any discrepancies or issues.\nStay updated on industry trends and best practices in data analysis and visualization techniques.\nBachelor's degree in Computer Science, Statistics, Mathematics, Economics, or related field. (Master's degree preferred)\nProven experience working as a data analyst or in a similar role.\nProficiency in data analysis tools and software such as Excel, SQL, Python, R, or Tableau.\nStrong analytical and problem-solving skills with the ability to translate complex data into actionable insights.\nExcellent communication and presentation skills, with the ability to effectively convey technical information to non-technical stakeholders.\nDetail-oriented with a commitment to accuracy and data integrity.\nAbility to work independently and collaboratively in a fast-paced environment.\nWhat's your expected monthly basic salary?"}

data/02_processed/74891040.json

@@ -0,0 +1 @@
+{"id": 74891040, "location": "petaling", "company": "agensi pekerjaan hays (malaysia) sdn bhd", "job_title": "principal data engineer", "salary": "$10000.0 - $14000.0 p.m.", "description": "Take ownership of the end-to-end data solution development and operation, including modelling, ETL, scripting, visualisation, testing, deployment, etc.\nDrive maturity of data infrastructure and architecture by designing, implementing, and maintaining the ingestion, ETL, storage, consumption, archival and disposal process and structure for product and organisation.\nBuild data products, including APIs, visualisation tools, analytics and machine learning platforms.\nUtilise ETL to provide actionable insights and intelligence, serving both product offerings and internal data consumption for business functions.\nCoach and mentor the junior team members to develop and improve both technical and non-technical capabilities.\nAt least 3 years of experience in a Data Engineering role.\nStrong command of SQL, Python, Data Modelling, Data Warehousing, ETL/ELT.\nStrong foundation in cloud data services such as AWS Redshift, Azure Synapse, BigQuery, Snowflakes, etc.\nA competitive salary based on performance and output.\nExcellent career support and leadership training.\nExposure to the growing fintech industry and room to grow at an elevated rate."}

data/02_processed/74915330.json

@@ -0,0 +1 @@
+{"id": 74915330, "location": "kuala lumpur", "company": "inter island manpower pte ltd", "job_title": "data engineer (python)", "salary": "RM\u00a06,000 \u2013 RM\u00a07,000 per month", "description": "Bachelor's or Master's degree in Computer Science, Engineering, or related fields.\n1-3 years as a Data Engineer with expertise in data engineering concepts.\nHands-on experience with Azure or AWS cloud services, Python/Java/Scala, and data processing frameworks.\nStrong knowledge of data modeling, database technologies, and problem-solving skills.\nExcellent communication, collaboration, and adaptability to fast-paced environments.\nDesign, develop, and maintain data pipelines for diverse sources into data lake/warehouse.\nCollaborate with data scientists for analysis and reporting needs.\nResearch and evaluate data sources for property development.\nDevelop web-scraping scripts and automate pipeline deployment.\nUtilize Azure, AWS, and relevant tech for data solutions.\nIntegrate data between systems/platforms, monitor, and optimize pipelines.\nEnsure data security, compliance, and quality while staying updated on data tech trends.\nHow many years' experience do you have as a Data Engineer?\nWhich of the following programming languages are you experienced in?"}

data/02_processed/74923865.json

@@ -0,0 +1 @@
+{"id": 74923865, "location": "petaling", "company": "monkeycloud media sdn. bhd.", "job_title": "data analyst", "salary": "RM\u00a03,000 \u2013 RM\u00a04,500 per month", "description": "Collect and compile data from different sources, including databases, and spreadsheets.\nFiltering and cleaning data to ensure accuracy.\nAnalyze data to identify trends, patterns, and anomalies, and provide actionable insights to support in various team decision-making.\nDevelop and maintain dashboards and reports to track key project performance metrics, including progress and target achievement.\nConduct ad-hoc data analysis to address specific project management questions or issues.\nAssist in the preparation of presentations and reports for project status updates and executive-level reporting.\nDiploma or Degree in any related field\nExperience in data analysis, business analysis or similar role, with a strong focus on data manipulation and analysis is preferred.\nFamiliarity with data analysis tools and concepts, with experience in Power BI being a strong advantage.\n2-3 years working experience as a Business Intelligence Analyst/Data Analyst or any position that related to data and business analytics with strong statistical background.\nPreferably with prior experience in the FMCG industry.\nExperience in data analysis, business analysis or similar role, with a strong focus on data manipulation and analysis is preferred.\nProficiency in data analysis tools and software, such as Power BI, Microsoft Excel (Pivot, Macro) or similar tools.\nStrong organizational skills and the ability to manage multiple tasks and priorities simultaneously.\nA collaborative and team-oriented mindset with the ability to work effectively in a cross-functional environment.\nExcellent communication and presentation skills, with the ability to convey complex information clearly and effectively.\nStrong analytical and problem-solving skills, with the ability to translate data into actionable insights.\nMonday to Friday\n : 8.30a.m to 5.30p.m\u00a0\nHow many years' experience do you have as a data analyst?\nHow many years' experience do you have as a Power BI Consultant?\nHow would you rate your Mandarin language skills?\nWhich of the following data analytics tools are you experienced with?\nWhich of the following Relational Database Management Systems (RDBMS) are you experienced with?"}

data/02_processed/74946043.json

@@ -0,0 +1 @@
+{"id": 74946043, "location": "bandar tasik selatan", "company": "kk group of companies", "job_title": "data engineer lead", "salary": "RM\u00a07,000 \u2013 RM\u00a08,000 per month", "description": "Making large and/or complex data more accessible, understandable and usable\nTransforming, improving and integrating data, depending on the business requirements\nDelivering the data in a useful and appealing way to users\nWork with noisy, dirty, and unstructured data, data cleansing, scraping unstructured data, and converting into structured data\nBuild and maintain data pipelines, ETL, databases and applications to acquire, store and manage data\nChoose, design, test and implement data software and solutions for projects and enterprise\nSelecting and integrating any Big Data tools and frameworks required to provide requested capabilities\nMonitoring performance and advising necessary infrastructure changes to optimize performance and cost\nArchitect, develop and implement end-to-end automated data flows for data and analytical needs\nPerform data profiling, data quality assessment, data cleansing and data transformation\nRecommend appropriate solutions architecture that is highly scalable based on business requirement\nCreate date tools for data scientists and data visualization team members\nBuilds the infrastructure required for optimal extraction, transformation, and loading for data from a wide variety of data sources using SQL, noSQL and cloud (AWS / Azure / Google / Alibaba / DigitalOcean) technologies\nBenchmark, evaluate and improve the scalability, robustness, efficiency and performance of big data platforms and applications\nEngage in the delivery of analytics assignments for business users involving, amongst others; data preparation, data visualization, creating business insights etc\nContribute and facilitate workshops/interviews bridging the gap between data analytics and business outcomes\nProvide strategic and tactical advice to business units by translating analytical solutions into out-of-the-box execution recommendations addressing business objectives and impacting bottom line results\nCombine your technical expertise with the ability to architect, deploy and oversee integrated analytics/business intelligence/technology solutions\nBuild case for analytics through need analysis and business case (quantify likely benefits)\nConsult with leaders of business teams and manage expectations, meet project and deliverable timelines, ensure accurate and timely completion of deliverables\nCollaborate with several other teams and roles across the organization on requirements gathering and solution deployment\nMake analytical product, service or process decisions likely to impact multiple groups of employees and/or customers (internal or external)\nManage technology solution portfolios, and identifying new technology approaches to solve business problems with a strong focus on leveraging enterprise data\nContribute to the building and management of the analytics infrastructure\nWork with management team on staffing needs, strategic technology trend identification, monitoring project progress and completion, issue resolution and technical solutions related to data, data flow and advising on algorithm deployment\nCollaborate with analytics practitioners, business analysts and methodologists to design and oversee the development of working prototypes of technical solutions\nTechnical support and lead for the Data Engineer team\nWork with project teams to assist with data-related technical issues and support their data infrastructure needs\nWhat's your expected monthly basic salary?\nHow many years' experience do you have as a Data Engineer?\nHow many years' experience do you have as a Tableau Specialist?\nHow many years' experience do you have as a Power BI Developer?\nHave you completed an Amazon Web Services Developer certification?\nHave you completed a Cloudera Certified Professional Data Engineer certification?\nHow many years' experience do you have as an ETL Data Warehouse Programmer?\nHow much notice are you required to give your current employer?"}

data/02_processed/74976890.json

@@ -0,0 +1 @@
+{"id": 74976890, "location": "kuala lumpur", "company": "alipay malaysia sdn. bhd.", "job_title": "risk operation data analyst ( online payment)", "salary": "RM\u00a010,000 \u2013 RM\u00a015,000 per month", "description": "What's your expected monthly basic salary?\nWhich of the following languages are you fluent in?\nHow many years' experience do you have as an Operational Risk Analyst?"}

data/02_processed/metadata/job_vacancy_metadata.json

@@ -1 +1 @@
-{"data engineer": ["data analysts/ data engineers (bangsar south- kerinchi lrt)", "data engineer"], "data scientist": [], "data analyst": ["data analysts/ data engineers (bangsar south- kerinchi lrt)", "junior data analyst", "data analyst executive", "e-commerce data analyst", "data analyst"], "machine learning engineer": ["machine learning engineer (artificial intelligence)", "machine learning engineer (ai)"]}
+{"data engineer": ["senior business intelligence developer / data engineer", "data engineer (python)", "big data engineer - cloud", "principal data engineer", "data engineer lead", "data engineer", "data engineer/data cloud engineer/salesforce", "manager - data engineer", "data analysts/ data engineers (bangsar south- kerinchi lrt)"], "data scientist": ["senior data scientist (full stack)", "data scientist", "data scientist assistant manager"], "data analyst": ["e-commerce data analyst", "risk operation data analyst ( online payment)", "data analyst/scientist", "data analyst (e-commerce)", "senior data analyst", "data analyst (intern)", "junior data analyst", "data analyst", "data analyst - internal audit", "data analyst internship", "data analyst executive", "data analysts/ data engineers (bangsar south- kerinchi lrt)"], "machine learning engineer": ["machine learning engineer (artificial intelligence)", "machine learning engineer (ai)"]}

notebooks/ETL_for_Kaggle_dataset.ipynb

@@ -1,5 +1,12 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The main purpose of this notebook is to parse the file with the scraped jobs into a json file per job."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 1,
@@ -46,7 +53,6 @@
     "conf_file = \".env\"\n",
     "\n",
     "data_dir = os.path.join(\"data\", \"01_raw\")\n",
-    "data_file = \"data_scientist_jobstreet_scraped_v2 2.csv\"\n",
     "\n",
     "res_dir = os.path.join(\"data\", \"02_processed\")\n",
     "\n",
@@ -75,37 +81,70 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "data_path = os.path.join(data_dir, data_file)\n",
-    "\n",
-    "data_df = pd.read_csv(data_path, sep=\",\")"
+    "# Get list of file names in ingestion folder\n",
+    "file_names = [file for file in os.listdir(data_dir) if \".csv\" in file]"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 6,
    "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['data_scientist_jobstreet_scraped_v2 2.csv',\n",
+       " 'data_jobs_scraped_from_jobstreet_2024-04-07.csv']"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "file_names"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "columns = ['job_id', 'job_title', 'company', 'location', 'salary', 'description']\n",
+    "\n",
+    "for file_name in file_names:\n",
+    "    data_path = os.path.join(data_dir, file_name)\n",
+    "    if 'data_df' not in locals():\n",
+    "        data_df = pd.read_csv(data_path, sep=\",\")[columns]\n",
+    "    else:\n",
+    "        data_file = pd.read_csv(data_path, sep=\",\")[columns]\n",
+    "        data_df = pd.concat([data_df, data_file])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "<class 'pandas.core.frame.DataFrame'>\n",
-      "RangeIndex: 606 entries, 0 to 605\n",
-      "Data columns (total 10 columns):\n",
-      " #   Column        Non-Null Count  Dtype \n",
-      "---  ------        --------------  ----- \n",
-      " 0   Unnamed: 0    606 non-null    int64 \n",
-      " 1   job_id        606 non-null    int64 \n",
-      " 2   job_title     606 non-null    object\n",
-      " 3   company       606 non-null    object\n",
-      " 4   descriptions  588 non-null    object\n",
-      " 5   location      606 non-null    object\n",
-      " 6   category      606 non-null    object\n",
-      " 7   subcategory   606 non-null    object\n",
-      " 8   type          606 non-null    object\n",
-      " 9   salary        198 non-null    object\n",
-      "dtypes: int64(2), object(8)\n",
-      "memory usage: 47.5+ KB\n"
+      "Index: 635 entries, 0 to 28\n",
+      "Data columns (total 6 columns):\n",
+      " #   Column       Non-Null Count  Dtype \n",
+      "---  ------       --------------  ----- \n",
+      " 0   job_id       635 non-null    int64 \n",
+      " 1   job_title    635 non-null    object\n",
+      " 2   company      635 non-null    object\n",
+      " 3   location     635 non-null    object\n",
+      " 4   salary       227 non-null    object\n",
+      " 5   description  617 non-null    object\n",
+      "dtypes: int64(1), object(5)\n",
+      "memory usage: 34.7+ KB\n"
      ]
     }
    ],
@@ -115,7 +154,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -139,130 +178,92 @@
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
-       "      <th>Unnamed: 0</th>\n",
        "      <th>job_id</th>\n",
        "      <th>job_title</th>\n",
        "      <th>company</th>\n",
-       "      <th>descriptions</th>\n",
        "      <th>location</th>\n",
-       "      <th>category</th>\n",
-       "      <th>subcategory</th>\n",
-       "      <th>type</th>\n",
        "      <th>salary</th>\n",
+       "      <th>description</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
-       "      <th>381</th>\n",
-       "      <td>381</td>\n",
-       "      <td>72235414</td>\n",
-       "      <td>Data Scientist (Credit)</td>\n",
-       "      <td>Michael Page International (Malaysia) Sdn Bhd</td>\n",
-       "      <td>Lead requirement gathering discussions for use...</td>\n",
+       "      <th>167</th>\n",
+       "      <td>72376611</td>\n",
+       "      <td>Head of Advanced Data Analytics</td>\n",
+       "      <td>Agensi Pekerjaan Hays (Malaysia) Sdn Bhd</td>\n",
        "      <td>Kuala Lumpur</td>\n",
-       "      <td>Banking &amp; Financial Services</td>\n",
-       "      <td>Analysis &amp; Reporting</td>\n",
-       "      <td>Full time</td>\n",
-       "      <td>NaN</td>\n",
+       "      <td>RM 20,000 – RM 30,000 per month</td>\n",
+       "      <td>Being the point-of-contact for advanced analyt...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>469</th>\n",
-       "      <td>469</td>\n",
-       "      <td>72700478</td>\n",
-       "      <td>RPA Manager (UiPath)</td>\n",
-       "      <td>Ambition Group Malaysia Sdn Bhd</td>\n",
-       "      <td>Develop and execute a comprehensive UiPath aut...</td>\n",
+       "      <th>101</th>\n",
+       "      <td>72761527</td>\n",
+       "      <td>Data Engineer</td>\n",
+       "      <td>ANHSIN TECHNOLOGY SDN BHD</td>\n",
        "      <td>Kuala Lumpur</td>\n",
-       "      <td>Information &amp; Communication Technology</td>\n",
-       "      <td>Team Leaders</td>\n",
-       "      <td>Full time</td>\n",
-       "      <td>RM 7,000 – RM 10,000 per month</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Design, develop, and maintain scalable and rob...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>167</th>\n",
-       "      <td>167</td>\n",
-       "      <td>72376611</td>\n",
-       "      <td>Head of Advanced Data Analytics</td>\n",
-       "      <td>Agensi Pekerjaan Hays (Malaysia) Sdn Bhd</td>\n",
-       "      <td>Being the point-of-contact for advanced analyt...</td>\n",
+       "      <th>376</th>\n",
+       "      <td>72531370</td>\n",
+       "      <td>ERP Analyst / Application Analyst</td>\n",
+       "      <td>Amazon Papyrus (M) Sdn Bhd</td>\n",
        "      <td>Kuala Lumpur</td>\n",
-       "      <td>Science &amp; Technology</td>\n",
-       "      <td>Mathematics, Statistics &amp; Information Sciences</td>\n",
-       "      <td>Full time</td>\n",
-       "      <td>RM 20,000 – RM 30,000 per month</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>Based in Malaysia, Work from home arrangement\\...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>314</th>\n",
-       "      <td>314</td>\n",
-       "      <td>72750107</td>\n",
-       "      <td>IT Business Analyst</td>\n",
-       "      <td>Mewah Group</td>\n",
-       "      <td>To develop new software applications and custo...</td>\n",
-       "      <td>Klang District</td>\n",
-       "      <td>Information &amp; Communication Technology</td>\n",
-       "      <td>Business/Systems Analysts</td>\n",
-       "      <td>Full time</td>\n",
+       "      <th>430</th>\n",
+       "      <td>72683703</td>\n",
+       "      <td>Data Engineer</td>\n",
+       "      <td>DATABRICKS TECH SDN. BHD.</td>\n",
+       "      <td>Bukit Jalil</td>\n",
        "      <td>NaN</td>\n",
+       "      <td>Maintain, backup, secure and transform data fr...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>5</th>\n",
-       "      <td>5</td>\n",
-       "      <td>72762762</td>\n",
-       "      <td>Senior Engineer / Engineer - Data Management &amp;...</td>\n",
-       "      <td>B. Braun Medical Industries Sdn. Bhd.</td>\n",
-       "      <td>Build, maintain and utilize a variety of datab...</td>\n",
-       "      <td>Penang</td>\n",
-       "      <td>Engineering</td>\n",
-       "      <td>Other</td>\n",
-       "      <td>Full time</td>\n",
+       "      <th>161</th>\n",
+       "      <td>72403257</td>\n",
+       "      <td>Data Engineer</td>\n",
+       "      <td>KINESSO MALAYSIA SDN. BHD. (fka Mediabrands Gl...</td>\n",
+       "      <td>Petaling</td>\n",
        "      <td>NaN</td>\n",
+       "      <td>Perform daily, weekly and monthly monitoring o...</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
-       "     Unnamed: 0    job_id                                          job_title  \\\n",
-       "381         381  72235414                            Data Scientist (Credit)   \n",
-       "469         469  72700478                               RPA Manager (UiPath)   \n",
-       "167         167  72376611                    Head of Advanced Data Analytics   \n",
-       "314         314  72750107                                IT Business Analyst   \n",
-       "5             5  72762762  Senior Engineer / Engineer - Data Management &...   \n",
-       "\n",
-       "                                           company  \\\n",
-       "381  Michael Page International (Malaysia) Sdn Bhd   \n",
-       "469                Ambition Group Malaysia Sdn Bhd   \n",
-       "167       Agensi Pekerjaan Hays (Malaysia) Sdn Bhd   \n",
-       "314                                    Mewah Group   \n",
-       "5            B. Braun Medical Industries Sdn. Bhd.   \n",
+       "       job_id                          job_title  \\\n",
+       "167  72376611    Head of Advanced Data Analytics   \n",
+       "101  72761527                      Data Engineer   \n",
+       "376  72531370  ERP Analyst / Application Analyst   \n",
+       "430  72683703                      Data Engineer   \n",
+       "161  72403257                      Data Engineer   \n",
        "\n",
-       "                                          descriptions        location  \\\n",
-       "381  Lead requirement gathering discussions for use...    Kuala Lumpur   \n",
-       "469  Develop and execute a comprehensive UiPath aut...    Kuala Lumpur   \n",
-       "167  Being the point-of-contact for advanced analyt...    Kuala Lumpur   \n",
-       "314  To develop new software applications and custo...  Klang District   \n",
-       "5    Build, maintain and utilize a variety of datab...          Penang   \n",
+       "                                               company      location  \\\n",
+       "167           Agensi Pekerjaan Hays (Malaysia) Sdn Bhd  Kuala Lumpur   \n",
+       "101                          ANHSIN TECHNOLOGY SDN BHD  Kuala Lumpur   \n",
+       "376                         Amazon Papyrus (M) Sdn Bhd  Kuala Lumpur   \n",
+       "430                          DATABRICKS TECH SDN. BHD.   Bukit Jalil   \n",
+       "161  KINESSO MALAYSIA SDN. BHD. (fka Mediabrands Gl...      Petaling   \n",
        "\n",
-       "                                   category  \\\n",
-       "381            Banking & Financial Services   \n",
-       "469  Information & Communication Technology   \n",
-       "167                    Science & Technology   \n",
-       "314  Information & Communication Technology   \n",
-       "5                               Engineering   \n",
+       "                              salary  \\\n",
+       "167  RM 20,000 – RM 30,000 per month   \n",
+       "101                              NaN   \n",
+       "376                              NaN   \n",
+       "430                              NaN   \n",
+       "161                              NaN   \n",
        "\n",
-       "                                        subcategory       type  \\\n",
-       "381                            Analysis & Reporting  Full time   \n",
-       "469                                    Team Leaders  Full time   \n",
-       "167  Mathematics, Statistics & Information Sciences  Full time   \n",
-       "314                       Business/Systems Analysts  Full time   \n",
-       "5                                             Other  Full time   \n",
-       "\n",
-       "                              salary  \n",
-       "381                              NaN  \n",
-       "469   RM 7,000 – RM 10,000 per month  \n",
-       "167  RM 20,000 – RM 30,000 per month  \n",
-       "314                              NaN  \n",
-       "5                                NaN  "
+       "                                           description  \n",
+       "167  Being the point-of-contact for advanced analyt...  \n",
+       "101  Design, develop, and maintain scalable and rob...  \n",
+       "376  Based in Malaysia, Work from home arrangement\\...  \n",
+       "430  Maintain, backup, secure and transform data fr...  \n",
+       "161  Perform daily, weekly and monthly monitoring o...  "
       ]
      },
      "metadata": {},
@@ -275,25 +276,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
-    "data_clean_df = data_df[['job_id', 'location', 'company', 'job_title', 'salary', 'descriptions']].dropna().drop_duplicates().copy()"
+    "data_clean_df = data_df[['job_id', 'location', 'company', 'job_title', 'salary', 'description']].dropna().drop_duplicates().copy()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "32"
+       "61"
       ]
      },
-     "execution_count": 9,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -304,7 +305,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -334,14 +335,29 @@
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
-       "      <th>4</th>\n",
+       "      <th>5</th>\n",
        "      <td>Data Analyst</td>\n",
-       "      <td>5</td>\n",
+       "      <td>10</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>7</th>\n",
+       "      <th>13</th>\n",
        "      <td>Data Engineer</td>\n",
-       "      <td>4</td>\n",
+       "      <td>6</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>Junior Data Analyst</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>Data Scientist</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>Machine Learning Engineer (Artificial Intellig...</td>\n",
+       "      <td>2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
@@ -349,112 +365,182 @@
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>14</th>\n",
+       "      <th>31</th>\n",
+       "      <td>MicroStrategy Developers (MNC)</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
        "      <td>Intern - Data Science</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>23</th>\n",
-       "      <td>System Analyst</td>\n",
+       "      <th>27</th>\n",
+       "      <td>Machine Learning Engineer (AI)</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>22</th>\n",
-       "      <td>Scientist (Bioinformatics)</td>\n",
+       "      <th>29</th>\n",
+       "      <td>Manager - Data Engineer</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>21</th>\n",
-       "      <td>RPA Manager (UiPath)</td>\n",
+       "      <th>30</th>\n",
+       "      <td>Manager, RPA Developer</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>20</th>\n",
+       "      <th>32</th>\n",
+       "      <td>Principal Data Engineer</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>Head of Advanced Data Analytics</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>33</th>\n",
        "      <td>Principal Engineer, Automation Project</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>19</th>\n",
-       "      <td>MicroStrategy Developers (MNC)</td>\n",
+       "      <th>34</th>\n",
+       "      <td>RPA Manager (UiPath)</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>18</th>\n",
-       "      <td>Manager, RPA Developer</td>\n",
+       "      <th>35</th>\n",
+       "      <td>Risk Operation Data Analyst ( Online Payment)</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>17</th>\n",
-       "      <td>Machine Learning Engineer (Artificial Intellig...</td>\n",
+       "      <th>36</th>\n",
+       "      <td>Scientist (Bioinformatics)</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>16</th>\n",
-       "      <td>Machine Learning Engineer (AI)</td>\n",
+       "      <th>37</th>\n",
+       "      <td>Senior Business Intelligence Developer / Data ...</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>15</th>\n",
-       "      <td>Junior Data Analyst</td>\n",
+       "      <th>38</th>\n",
+       "      <td>Senior Data Analyst</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>12</th>\n",
-       "      <td>Head of Advanced Data Analytics</td>\n",
+       "      <th>39</th>\n",
+       "      <td>Senior Data Scientist (Full Stack)</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>13</th>\n",
+       "      <th>40</th>\n",
+       "      <td>System Analyst</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
        "      <td>IT Business Analyst</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>BUSINESS SYSTEM ANALYST</td>\n",
+       "      <th>21</th>\n",
+       "      <td>E-commerce Data Analyst</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>11</th>\n",
+       "      <th>22</th>\n",
        "      <td>ERP System Analyst</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
-       "      <td>E-commerce Data Analyst</td>\n",
+       "      <td>Data Analyst Internship</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>9</th>\n",
-       "      <td>Database Administrator (DBA)</td>\n",
+       "      <th>2</th>\n",
+       "      <td>Big Data Engineer - Cloud</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>8</th>\n",
-       "      <td>Database Administrator</td>\n",
+       "      <th>3</th>\n",
+       "      <td>Contract Business Administration Analyst</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>DATA ANALYST</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6</th>\n",
-       "      <td>Data Analysts/ Data Engineers (Bangsar South- ...</td>\n",
+       "      <td>Data Analyst (E-Commerce)</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>5</th>\n",
+       "      <th>7</th>\n",
+       "      <td>Data Analyst (Intern)</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>Data Analyst - Internal Audit</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
        "      <td>Data Analyst Executive</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>DATA ANALYST</td>\n",
+       "      <th>11</th>\n",
+       "      <td>Data Analyst/Scientist</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>Contract Business Administration Analyst</td>\n",
+       "      <th>1</th>\n",
+       "      <td>BUSINESS SYSTEM ANALYST</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>24</th>\n",
+       "      <th>12</th>\n",
+       "      <td>Data Analysts/ Data Engineers (Bangsar South- ...</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>Data Engineer (Python)</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>Data Engineer Lead</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>Data Engineer/Data Cloud Engineer/Salesforce</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>Data Scientist Assistant Manager</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>Database Administrator</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>Database Administrator (DBA)</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>41</th>\n",
        "      <td>System Analyst - IT</td>\n",
        "      <td>1</td>\n",
        "    </tr>\n",
@@ -464,34 +550,51 @@
       ],
       "text/plain": [
        "                                            job_title  job_id\n",
-       "4                                        Data Analyst       5\n",
-       "7                                       Data Engineer       4\n",
+       "5                                        Data Analyst      10\n",
+       "13                                      Data Engineer       6\n",
+       "26                                Junior Data Analyst       3\n",
+       "17                                     Data Scientist       3\n",
+       "28  Machine Learning Engineer (Artificial Intellig...       2\n",
        "0                     BI Developer - Immediate Hiring       1\n",
-       "14                              Intern - Data Science       1\n",
-       "23                                     System Analyst       1\n",
-       "22                         Scientist (Bioinformatics)       1\n",
-       "21                               RPA Manager (UiPath)       1\n",
-       "20             Principal Engineer, Automation Project       1\n",
-       "19                     MicroStrategy Developers (MNC)       1\n",
-       "18                             Manager, RPA Developer       1\n",
-       "17  Machine Learning Engineer (Artificial Intellig...       1\n",
-       "16                     Machine Learning Engineer (AI)       1\n",
-       "15                                Junior Data Analyst       1\n",
-       "12                    Head of Advanced Data Analytics       1\n",
-       "13                                IT Business Analyst       1\n",
+       "31                     MicroStrategy Developers (MNC)       1\n",
+       "25                              Intern - Data Science       1\n",
+       "27                     Machine Learning Engineer (AI)       1\n",
+       "29                            Manager - Data Engineer       1\n",
+       "30                             Manager, RPA Developer       1\n",
+       "32                            Principal Data Engineer       1\n",
+       "23                    Head of Advanced Data Analytics       1\n",
+       "33             Principal Engineer, Automation Project       1\n",
+       "34                               RPA Manager (UiPath)       1\n",
+       "35      Risk Operation Data Analyst ( Online Payment)       1\n",
+       "36                         Scientist (Bioinformatics)       1\n",
+       "37  Senior Business Intelligence Developer / Data ...       1\n",
+       "38                                Senior Data Analyst       1\n",
+       "39                 Senior Data Scientist (Full Stack)       1\n",
+       "40                                     System Analyst       1\n",
+       "24                                IT Business Analyst       1\n",
+       "21                            E-commerce Data Analyst       1\n",
+       "22                                 ERP System Analyst       1\n",
+       "10                            Data Analyst Internship       1\n",
+       "2                           Big Data Engineer - Cloud       1\n",
+       "3            Contract Business Administration Analyst       1\n",
+       "4                                        DATA ANALYST       1\n",
+       "6                           Data Analyst (E-Commerce)       1\n",
+       "7                               Data Analyst (Intern)       1\n",
+       "8                       Data Analyst - Internal Audit       1\n",
+       "9                              Data Analyst Executive       1\n",
+       "11                             Data Analyst/Scientist       1\n",
        "1                             BUSINESS SYSTEM ANALYST       1\n",
-       "11                                 ERP System Analyst       1\n",
-       "10                            E-commerce Data Analyst       1\n",
-       "9                        Database Administrator (DBA)       1\n",
-       "8                              Database Administrator       1\n",
-       "6   Data Analysts/ Data Engineers (Bangsar South- ...       1\n",
-       "5                              Data Analyst Executive       1\n",
-       "3                                        DATA ANALYST       1\n",
-       "2            Contract Business Administration Analyst       1\n",
-       "24                                System Analyst - IT       1"
+       "12  Data Analysts/ Data Engineers (Bangsar South- ...       1\n",
+       "14                             Data Engineer (Python)       1\n",
+       "15                                 Data Engineer Lead       1\n",
+       "16       Data Engineer/Data Cloud Engineer/Salesforce       1\n",
+       "18                   Data Scientist Assistant Manager       1\n",
+       "19                             Database Administrator       1\n",
+       "20                       Database Administrator (DBA)       1\n",
+       "41                                System Analyst - IT       1"
       ]
      },
-     "execution_count": 10,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -502,7 +605,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -514,7 +617,7 @@
     "        \"company\": row['company'].lower(),\n",
     "        \"job_title\": row['job_title'].lower(),\n",
     "        \"salary\": row['salary'],\n",
-    "        \"description\": row['descriptions'],\n",
+    "        \"description\": row['description'],\n",
     "    }\n",
     "\n",
     "    res_file = \"{job_id}.json\".format(job_id=row['job_id'])\n",
@@ -526,25 +629,41 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "{'data engineer': ['data analysts/ data engineers (bangsar south- kerinchi lrt)',\n",
-       "  'data engineer'],\n",
-       " 'data scientist': [],\n",
-       " 'data analyst': ['data analysts/ data engineers (bangsar south- kerinchi lrt)',\n",
+       "{'data engineer': ['senior business intelligence developer / data engineer',\n",
+       "  'data engineer (python)',\n",
+       "  'big data engineer - cloud',\n",
+       "  'principal data engineer',\n",
+       "  'data engineer lead',\n",
+       "  'data engineer',\n",
+       "  'data engineer/data cloud engineer/salesforce',\n",
+       "  'manager - data engineer',\n",
+       "  'data analysts/ data engineers (bangsar south- kerinchi lrt)'],\n",
+       " 'data scientist': ['senior data scientist (full stack)',\n",
+       "  'data scientist',\n",
+       "  'data scientist assistant manager'],\n",
+       " 'data analyst': ['e-commerce data analyst',\n",
+       "  'risk operation data analyst ( online payment)',\n",
+       "  'data analyst/scientist',\n",
+       "  'data analyst (e-commerce)',\n",
+       "  'senior data analyst',\n",
+       "  'data analyst (intern)',\n",
        "  'junior data analyst',\n",
+       "  'data analyst',\n",
+       "  'data analyst - internal audit',\n",
+       "  'data analyst internship',\n",
        "  'data analyst executive',\n",
-       "  'e-commerce data analyst',\n",
-       "  'data analyst'],\n",
+       "  'data analysts/ data engineers (bangsar south- kerinchi lrt)'],\n",
        " 'machine learning engineer': ['machine learning engineer (artificial intelligence)',\n",
        "  'machine learning engineer (ai)']}"
       ]
      },
-     "execution_count": 12,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -564,7 +683,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [],
    "source": [

notebooks/jobstreet-job-scraping.ipynb

@@ -0,0 +1 @@
+{"cells":[{"cell_type":"markdown","metadata":{},"source":["The main goal of this notebook is to scrape Data Scientist job vacancy from [Jobstreet Malaysia](https://www.jobstreet.com.my/)(a job searching website).\n","\n","We'll be using the combination API and manual HTML scraping technique from the Jobstreet for our webscraping."]},{"cell_type":"code","execution_count":1,"metadata":{"execution":{"iopub.execute_input":"2024-04-07T09:21:59.667260Z","iopub.status.busy":"2024-04-07T09:21:59.666761Z","iopub.status.idle":"2024-04-07T09:22:00.422635Z","shell.execute_reply":"2024-04-07T09:22:00.421244Z","shell.execute_reply.started":"2024-04-07T09:21:59.667217Z"},"tags":[],"trusted":true},"outputs":[],"source":["import os\n","import requests\n","from bs4 import BeautifulSoup\n","from resume_worth.utils.utils import get_params\n","import pandas as pd\n","pd.set_option('display.max_colwidth', 200)"]},{"cell_type":"code","execution_count":2,"metadata":{},"outputs":[{"data":{"text/plain":["'/Users/luisrodrigues/Documents/Projects/PERSONAL/resume-worth'"]},"execution_count":2,"metadata":{},"output_type":"execute_result"}],"source":["# Change the current working directory to the pachage root\n","# That's step is due to the way settings.py is defined\n","ROOT_DIR = os.path.join(*os.path.split(os.getcwd())[:-1])\n","os.chdir(ROOT_DIR)\n","os.getcwd()"]},{"cell_type":"code","execution_count":3,"metadata":{},"outputs":[],"source":["max_pages = 15\n","# api url taken from Network -> Header \n","api_url = 'https://www.jobstreet.com.my/api/chalice-search/v4/search?siteKey=MY-Main&sourcesystem=houston&userqueryid=2b00edd417ec163434fca9421e24c97a-7428735&userid=e56c7e89-1d1a-42b9-b7a5-37c12653d6b9&usersessionid=e56c7e89-1d1a-42b9-b7a5-37c12653d6b9&eventCaptureSessionId=e56c7e89-1d1a-42b9-b7a5-37c12653d6b9&seekSelectAllPages=true&keywords=data+scientist&pageSize=99&include=seodata&locale=en-MY&solId=568d2fe8-e8ef-4998-8e24-3e1ccfb1348b'\n","\n","data_dir = os.path.join(\"data\", \"01_raw\")\n","ingestion_date = pd.to_datetime('today').strftime('%Y-%m-%d')\n","data_file = f\"data_jobs_scraped_from_jobstreet_{ingestion_date}.csv\""]},{"cell_type":"markdown","metadata":{},"source":["## Scrape available job articles"]},{"cell_type":"code","execution_count":4,"metadata":{"execution":{"iopub.execute_input":"2024-04-07T09:22:00.425276Z","iopub.status.busy":"2024-04-07T09:22:00.424497Z","iopub.status.idle":"2024-04-07T09:22:01.980509Z","shell.execute_reply":"2024-04-07T09:22:01.979252Z","shell.execute_reply.started":"2024-04-07T09:22:00.425240Z"},"tags":[],"trusted":true},"outputs":[],"source":["def scrape_article_ids(api_url, max_pages):\n","    job_id =[]\n","    titles = []\n","    companies = []\n","    locations = []\n","    categorys= []\n","    subCategorys= []\n","    job_types=[]\n","    salarys=[]\n","    for page_number in range(1, max_pages + 1):\n","        page_url = f'{api_url}&page={page_number}'\n","        \n","        # Send an HTTP request to the API endpoint\n","        response = requests.get(page_url)\n","        if response.status_code == 200:\n","            # Parse the JSON response\n","            data = response.json()\n","\n","            # Extract advertiser IDs from each item in the 'data' list\n","            for item in data['data']:\n","                #print(item)\n","                jid = item['id']\n","                title = item['title']\n","                company = item['advertiser'].get('description', '')\n","                location = item.get('location', '')\n","                category = item['classification'].get('description', '')\n","                subCategory= item['subClassification'].get('description', '')\n","                job_type = item.get('workType', '')\n","                salary = item.get('salary', '')\n","\n","                job_id.append(jid)\n","                titles.append(title)\n","                companies.append(company)\n","                locations.append(location)\n","                categorys.append(category)\n","                subCategorys.append(subCategory)\n","                job_types.append(job_type)\n","                salarys.append(salary)\n","                #print(f\"Job ID: {job_id}\")\n","\n","        else:\n","            print(f\"Failed to retrieve data from the API. Status Code: {response.status_code}\")\n","            break\n","\n","    return job_id, titles, companies, locations, categorys, subCategorys,job_types,salarys"]},{"cell_type":"code","execution_count":5,"metadata":{},"outputs":[],"source":["job_ids, titles, companies, locations, categorys, sub_categorys, job_types, salarys = scrape_article_ids(api_url, max_pages)\n","\n","jobs = []\n","for idx, job_id in enumerate(job_ids):\n","    job_info = {\n","        'job_id': job_ids[idx], \n","        'job_title': titles[idx], \n","        'company': companies[idx], \n","        'location': locations[idx], \n","        'category': categorys[idx], \n","        'sub_category': sub_categorys[idx], \n","        'job_type': job_types[idx], \n","        'salary': salarys[idx]\n","    }\n","    jobs.append(job_info)"]},{"cell_type":"code","execution_count":6,"metadata":{"execution":{"iopub.execute_input":"2024-04-07T09:24:27.229473Z","iopub.status.busy":"2024-04-07T09:24:27.228315Z","iopub.status.idle":"2024-04-07T09:24:27.246251Z","shell.execute_reply":"2024-04-07T09:24:27.244966Z","shell.execute_reply.started":"2024-04-07T09:24:27.229409Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Number of retrieved jobs: 1515\n"]}],"source":["print(f\"Number of retrieved jobs: {len(jobs)}\")"]},{"cell_type":"markdown","metadata":{},"source":["# Filter retrieved jobs by job title and salary availability"]},{"cell_type":"code","execution_count":7,"metadata":{},"outputs":[],"source":["params = get_params()\n","target_job_titles = params['job_titles']"]},{"cell_type":"code","execution_count":8,"metadata":{},"outputs":[{"data":{"text/plain":["['Data Engineer',\n"," 'Data Scientist',\n"," 'Data Analyst',\n"," 'Machine Learning Engineer']"]},"execution_count":8,"metadata":{},"output_type":"execute_result"}],"source":["target_job_titles"]},{"cell_type":"code","execution_count":9,"metadata":{},"outputs":[],"source":["target_jobs = []\n","for job in jobs:\n","    if len(job['salary']) > 0:\n","        for target_job_title in target_job_titles:\n","            if target_job_title.lower() in job['job_title'].lower():\n","                target_jobs.append(job)\n","                break"]},{"cell_type":"code","execution_count":10,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Number of filtered jobs: 31\n"]}],"source":["print(f\"Number of filtered jobs: {len(target_jobs)}\")"]},{"cell_type":"markdown","metadata":{},"source":["## Fetch job descriptions from individual job link"]},{"cell_type":"code","execution_count":11,"metadata":{"execution":{"iopub.execute_input":"2024-04-07T09:22:07.307243Z","iopub.status.busy":"2024-04-07T09:22:07.306836Z","iopub.status.idle":"2024-04-07T09:22:07.319745Z","shell.execute_reply":"2024-04-07T09:22:07.318194Z","shell.execute_reply.started":"2024-04-07T09:22:07.307211Z"},"tags":[],"trusted":true},"outputs":[],"source":["def fetch_job_article(job_id):\n","    article_url = f'https://www.jobstreet.com.my/job/{job_id}'\n","    response = requests.get(article_url)\n","    if response.status_code == 200:\n","        return response.text\n","    else:\n","        print(f\"Failed to retrieve job article. Status Code: {response.status_code}\")\n","        return None\n","\n","def extract_text_from_ul(html_content):\n","    soup = BeautifulSoup(html_content, 'html.parser')\n","    ul_tags = soup.find_all('ul')\n","    text_list = [ul.get_text(separator='\\n') for ul in ul_tags]\n","    return '\\n'.join(text_list)\n","\n","def scrape_and_store_text(job_id):\n","    job_article_content = fetch_job_article(job_id)\n","        \n","    if job_article_content:\n","        text_from_ul = extract_text_from_ul(job_article_content)\n","        return text_from_ul\n","    \n","    return ''"]},{"cell_type":"code","execution_count":12,"metadata":{"execution":{"iopub.execute_input":"2024-04-07T09:22:09.177829Z","iopub.status.busy":"2024-04-07T09:22:09.177433Z","iopub.status.idle":"2024-04-07T09:24:23.150504Z","shell.execute_reply":"2024-04-07T09:24:23.148399Z","shell.execute_reply.started":"2024-04-07T09:22:09.177798Z"},"tags":[],"trusted":true},"outputs":[],"source":["completed_jobs = []\n","for job in target_jobs:\n","    description = scrape_and_store_text(job['job_id'])\n","    if len(description) > 0:\n","        job['description'] = description\n","        completed_jobs.append(job)"]},{"cell_type":"code","execution_count":13,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Number of completly retrieved jobs: 29\n"]}],"source":["print(f\"Number of completly retrieved jobs: {len(completed_jobs)}\")"]},{"cell_type":"code","execution_count":14,"metadata":{},"outputs":[],"source":["result_df = pd.DataFrame(completed_jobs)"]},{"cell_type":"code","execution_count":15,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>job_id</th>\n","      <th>job_title</th>\n","      <th>company</th>\n","      <th>location</th>\n","      <th>category</th>\n","      <th>sub_category</th>\n","      <th>job_type</th>\n","      <th>salary</th>\n","      <th>description</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>74727704</td>\n","      <td>Machine Learning Engineer (Artificial Intelligence)</td>\n","      <td>Private Advertiser</td>\n","      <td>Kuala Lumpur</td>\n","      <td>Science &amp; Technology</td>\n","      <td>Mathematics, Statistics &amp; Information Sciences</td>\n","      <td>Full time</td>\n","      <td>RM 4,000 – RM 6,000 per month</td>\n","      <td>Design, develop, and deploy machine learning models and algorithms for complex and unique datasets, using various techniques such as mathematical modeling, scikit-learn, NLP, CNN, RNN, DL, RL, Tra...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>74850440</td>\n","      <td>Data Scientist</td>\n","      <td>PHENOMENAL UAM Sdn Bhd</td>\n","      <td>Kuala Lumpur</td>\n","      <td>Science &amp; Technology</td>\n","      <td>Mathematics, Statistics &amp; Information Sciences</td>\n","      <td>Full time</td>\n","      <td>RM 5,000 – RM 7,500 per month</td>\n","      <td>Utilizing machine learning techniques to analyze and interpret complex data sets.\\nDeveloping AI models and algorithms to solve specific business problems and improve operational efficiency.\\nColl...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>74946043</td>\n","      <td>Data Engineer Lead</td>\n","      <td>KK Group Of Companies</td>\n","      <td>Bandar Tasik Selatan</td>\n","      <td>Information &amp; Communication Technology</td>\n","      <td>Engineering - Software</td>\n","      <td>Full time</td>\n","      <td>RM 7,000 – RM 8,000 per month</td>\n","      <td>Making large and/or complex data more accessible, understandable and usable\\nTransforming, improving and integrating data, depending on the business requirements\\nDelivering the data in a useful a...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["     job_id                                            job_title  \\\n","0  74727704  Machine Learning Engineer (Artificial Intelligence)   \n","1  74850440                                       Data Scientist   \n","2  74946043                                   Data Engineer Lead   \n","\n","                  company              location  \\\n","0      Private Advertiser          Kuala Lumpur   \n","1  PHENOMENAL UAM Sdn Bhd          Kuala Lumpur   \n","2   KK Group Of Companies  Bandar Tasik Selatan   \n","\n","                                 category  \\\n","0                    Science & Technology   \n","1                    Science & Technology   \n","2  Information & Communication Technology   \n","\n","                                     sub_category   job_type  \\\n","0  Mathematics, Statistics & Information Sciences  Full time   \n","1  Mathematics, Statistics & Information Sciences  Full time   \n","2                          Engineering - Software  Full time   \n","\n","                          salary  \\\n","0  RM 4,000 – RM 6,000 per month   \n","1  RM 5,000 – RM 7,500 per month   \n","2  RM 7,000 – RM 8,000 per month   \n","\n","                                                                                                                                                                                               description  \n","0  Design, develop, and deploy machine learning models and algorithms for complex and unique datasets, using various techniques such as mathematical modeling, scikit-learn, NLP, CNN, RNN, DL, RL, Tra...  \n","1  Utilizing machine learning techniques to analyze and interpret complex data sets.\\nDeveloping AI models and algorithms to solve specific business problems and improve operational efficiency.\\nColl...  \n","2  Making large and/or complex data more accessible, understandable and usable\\nTransforming, improving and integrating data, depending on the business requirements\\nDelivering the data in a useful a...  "]},"execution_count":15,"metadata":{},"output_type":"execute_result"}],"source":["result_df.head(3)"]},{"cell_type":"code","execution_count":16,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>job_title</th>\n","      <th>job_id</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>1</th>\n","      <td>Data Analyst</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>Data Scientist</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>Data Engineer</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>Junior Data Analyst</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>0</th>\n","      <td>Big Data Engineer - Cloud</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>Data Scientist Assistant Manager</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>Senior Data Analyst</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>Senior Business Intelligence Developer / Data Engineer</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>Risk Operation Data Analyst ( Online Payment)</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>Principal Data Engineer</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>Manager - Data Engineer</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>Machine Learning Engineer (Artificial Intelligence)</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>Data Engineer/Data Cloud Engineer/Salesforce</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>Data Engineer Lead</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>Data Engineer (Python)</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>Data Analyst/Scientist</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>Data Analyst Internship</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Data Analyst - Internal Audit</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Data Analyst (Intern)</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Data Analyst (E-Commerce)</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>20</th>\n","      <td>Senior Data Scientist (Full Stack)</td>\n","      <td>1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                                                 job_title  job_id\n","1                                             Data Analyst       5\n","11                                          Data Scientist       3\n","7                                            Data Engineer       2\n","13                                     Junior Data Analyst       2\n","0                                Big Data Engineer - Cloud       1\n","12                        Data Scientist Assistant Manager       1\n","19                                     Senior Data Analyst       1\n","18  Senior Business Intelligence Developer / Data Engineer       1\n","17           Risk Operation Data Analyst ( Online Payment)       1\n","16                                 Principal Data Engineer       1\n","15                                 Manager - Data Engineer       1\n","14     Machine Learning Engineer (Artificial Intelligence)       1\n","10            Data Engineer/Data Cloud Engineer/Salesforce       1\n","9                                       Data Engineer Lead       1\n","8                                   Data Engineer (Python)       1\n","6                                   Data Analyst/Scientist       1\n","5                                  Data Analyst Internship       1\n","4                            Data Analyst - Internal Audit       1\n","3                                    Data Analyst (Intern)       1\n","2                                Data Analyst (E-Commerce)       1\n","20                      Senior Data Scientist (Full Stack)       1"]},"execution_count":16,"metadata":{},"output_type":"execute_result"}],"source":["result_df.groupby('job_title')[['job_id']].count().reset_index().sort_values(by=\"job_id\",ascending=False)"]},{"cell_type":"code","execution_count":17,"metadata":{},"outputs":[],"source":["data_path = os.path.join(data_dir, data_file)\n","\n","data_df = result_df.to_csv(data_path, sep=\",\")"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"datasetId":4284628,"sourceId":7654855,"sourceType":"datasetVersion"}],"dockerImageVersionId":30626,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.8"}},"nbformat":4,"nbformat_minor":4}

src/app/app.py

@@ -4,15 +4,19 @@ from resume_worth.pipelines.information_retrieval.pipeline import retrieve_top_j
 
 
 params = get_params()
-APP_TITLE = params['APP_TITLE']
-APP_DESCRIPTION = params['APP_DESCRIPTION']
-JOB_TITLES = params['JOB_TITLES']
-HOST = params['HOST']
-PORT = params['PORT']
+app_config = params['app_config']
+app_backend = params['app_backend']
+app_frontend = params['app_frontend']
 
 
-def salary_estimator(job_title, resume):
+def salary_estimator(job_title: str, resume: str):
 
+    if job_title is None:
+        return app_frontend['messages']['job_title_not_found']
+
+    if len(resume) < app_backend['min_resume_size']:
+        return app_frontend['messages']['salary_not_found']
+    
     salaries = retrieve_top_job_vacancy_info(job_title, resume)
 
     return salaries[0]
@@ -23,19 +27,19 @@ def run():
     app = gr.Interface(
         fn=salary_estimator, 
         inputs=[
-            gr.Radio(JOB_TITLES, label="Job Title"),
+            gr.Radio(app_frontend['jobs'], label="Job Title"),
             gr.Textbox(label="Resume", lines=10)
         ],
         outputs=[
             gr.Textbox(label="Estimate Salary", lines=1)
         ],
-        title=APP_TITLE,
-        description=APP_DESCRIPTION,
+        title=app_frontend['title'],
+        description=app_frontend['description'],
         allow_flagging="never"
     )
 
     # Use share=True to create a public link to share. This share link expires in 72 hours.
-    app.launch(server_name=HOST, server_port=int(PORT))
+    app.launch(server_name=app_config['host'], server_port=app_config['port'])
 
 
 if __name__ == "__main__":